Merge pull request #22 from bmaltais/dev

Update LoRA GUI
2023-01-01 14:15:41 -05:00 · 2023-01-01 14:15:41 -05:00 · 98f7dd5080
commit 98f7dd5080
parent cdf84e2f4b af46ce4c47
12 changed files with 1857 additions and 2657 deletions
--- a/README.md
+++ b/README.md
@ -12,18 +12,38 @@ You can find the dreambooth solution spercific [Dreambooth README](README_dreamb

 You can find the finetune solution spercific [Finetune README](README_finetune.md)

+## LoRA
+
+You can create LoRA network by running the dedicated GUI with:
+
+```
+python lora_gui.py
+```
+
+or via the all in one GUI:
+
+```
+python kahya_gui.py
+```
+
+Once you have created the LoRA network you can generate images via auto1111 by installing the extension found here: https://github.com/kohya-ss/sd-webui-additional-networks
+
 ## Change history

-* 12/30 (v19) update:
+* 2023/01/01 (v19.1) update:
+    - merge kohys_ss upstream code  updates
+    - rework Dreambooth LoRA GUI
+    - fix bug where LoRA network weights were not loaded to properly resume training
+* 2022/12/30 (v19) update:
    - support for LoRA network training in kohya_gui.py.
-* 12/23 (v18.8) update:
+* 2022/12/23 (v18.8) update:
    - Fix for conversion tool issue when the source was an sd1.x diffuser model
    - Other minor code and GUI fix
-* 12/22 (v18.7) update:
+* 2022/12/22 (v18.7) update:
    - Merge dreambooth and finetune is a common GUI
    - General bug fixes and code improvements
-* 12/21 (v18.6.1) update:
+* 2022/12/21 (v18.6.1) update:
    - fix issue with dataset balancing when the number of detected images in the folder is 0

-* 12/21 (v18.6) update:
+* 2022/12/21 (v18.6) update:
    - add optional GUI authentication support via: `python fine_tune.py --username=<name> --password=<password>`
--- a/bitsandbytes_windows/main.py
+++ b/bitsandbytes_windows/main.py
@ -4,7 +4,7 @@ extract factors the build is dependent on:
    [ ] TODO: Q - What if we have multiple GPUs of different makes?
 - CUDA version
 - Software:
-    - CPU-only: only CPU quantization functions (no optimizer, no matrix multipl)
+    - CPU-only: only CPU quantization functions (no optimizer, no matrix multiple)
    - CuBLAS-LT: full-build 8-bit optimizer
    - no CuBLAS-LT: no 8-bit matrix multiplication (`nomatmul`)

@ -44,7 +44,7 @@ def get_cuda_version(cuda, cudart_path):
    minor = (version-(major*1000))//10

    if major < 11:
-       print('CUDA SETUP: CUDA version lower than 11 are currenlty not supported for LLM.int8(). You will be only to use 8-bit optimizers and quantization routines!!')
+       print('CUDA SETUP: CUDA version lower than 11 are currently not supported for LLM.int8(). You will be only to use 8-bit optimizers and quantization routines!!')

    return f'{major}{minor}'

@ -163,4 +163,4 @@ def evaluate_cuda_setup():

    binary_name = get_binary_name()

-    return binary_name
+    return binary_name
--- a/dreambooth_gui.py
+++ b/dreambooth_gui.py
@ -15,6 +15,7 @@ from library.common_gui import (
    get_folder_path,
    remove_doublequote,
    get_file_path,
+    get_any_file_path,
    get_saveasfile_path,
 )
 from library.dreambooth_folder_creation_gui import (
@ -236,7 +237,7 @@ def train_model(
    seed,
    num_cpu_threads_per_process,
    cache_latent,
-    caption_extention,
+    caption_extension,
    enable_bucket,
    gradient_checkpointing,
    full_fp16,
@ -396,7 +397,8 @@ def train_model(
    run_cmd += f' --seed={seed}'
    run_cmd += f' --save_precision={save_precision}'
    run_cmd += f' --logging_dir={logging_dir}'
-    run_cmd += f' --caption_extention={caption_extention}'
+    if not caption_extension == '':
+        run_cmd += f' --caption_extension={caption_extension}'
    if not stop_text_encoder_training == 0:
        run_cmd += (
            f' --stop_text_encoder_training={stop_text_encoder_training}'
@ -542,7 +544,7 @@ def dreambooth_tab(
                document_symbol, elem_id='open_folder_small'
            )
            pretrained_model_name_or_path_fille.click(
-                get_file_path,
+                get_any_file_path,
                inputs=[pretrained_model_name_or_path_input],
                outputs=pretrained_model_name_or_path_input,
            )
--- a/examples/word_frequency.ps1
+++ b/examples/word_frequency.ps1
@ -1,12 +1,12 @@
-$txt_files_folder = "D:\dreambooth\training_twq\mad_hatter\all"
-$txt_prefix_to_ignore = "asd"
-$txt_postfix_ti_ignore = "asd"
+$txt_files_folder = "D:\dataset\metart_g1\img\100_asd girl"
+$txt_prefix_to_ignore = "asds"
+$txt_postfix_ti_ignore = "asds"

 # Should not need to touch anything below

 # (Get-Content $txt_files_folder"\*.txt" ).Replace(",", "") -Split '\W' | Group-Object -NoElement | Sort-Object -Descending -Property Count

-$combined_txt = Get-Content $txt_files_folder"\*.txt"
+$combined_txt = Get-Content $txt_files_folder"\*.cap"
 $combined_txt = $combined_txt.Replace(",", "")
 $combined_txt = $combined_txt.Replace("$txt_prefix_to_ignore", "")
 $combined_txt = $combined_txt.Replace("$txt_postfix_ti_ignore", "") -Split '\W' | Group-Object -NoElement | Sort-Object -Descending -Property Count
--- a/fine_tune.py
+++ b/fine_tune.py
--- a/finetune_gui.py
+++ b/finetune_gui.py
@ -9,6 +9,7 @@ import argparse
 from library.common_gui import (
    get_folder_path,
    get_file_path,
+    get_any_file_path,
    get_saveasfile_path,
 )
 from library.utilities import utilities_tab
@ -436,7 +437,7 @@ def finetune_tab():
                document_symbol, elem_id='open_folder_small'
            )
            pretrained_model_name_or_path_file.click(
-                get_file_path,
+                get_any_file_path,
                inputs=pretrained_model_name_or_path_input,
                outputs=pretrained_model_name_or_path_input,
            )
--- a/gen_img_diffusers.py
+++ b/gen_img_diffusers.py
@ -2486,9 +2486,9 @@ if __name__ == '__main__':
  parser.add_argument("--bf16", action='store_true', help='use bfloat16 / bfloat16を指定し省メモリ化する')
  parser.add_argument("--xformers", action='store_true', help='use xformers / xformersを使用し高速化する')
  parser.add_argument("--diffusers_xformers", action='store_true',
-                      help='use xformers by diffusers (Hypernetworks doen\'t work) / Diffusersでxformersを使用する（Hypernetwork利用不可）')
+                      help='use xformers by diffusers (Hypernetworks doesn\'t work) / Diffusersでxformersを使用する（Hypernetwork利用不可）')
  parser.add_argument("--opt_channels_last", action='store_true',
-                      help='set channels last option to model / モデルにchannles lastを指定し最適化する')
+                      help='set channels last option to model / モデルにchannels lastを指定し最適化する')
  parser.add_argument("--network_module", type=str, default=None, help='Hypernetwork module to use / Hypernetworkを使う時そのモジュール名')
  parser.add_argument("--network_weights", type=str, default=None, help='Hypernetwork weights to load / Hypernetworkの重み')
  parser.add_argument("--network_mul", type=float, default=1.0, help='Hypernetwork multiplier / Hypernetworkの効果の倍率')
@ -2514,4 +2514,4 @@ if __name__ == '__main__':
                      help="save 1st stage images for highres fix / highres fixの最初のステージの画像を保存する")

  args = parser.parse_args()
-  main(args)
+  main(args)
--- a/kohya_gui.py
+++ b/kohya_gui.py
@ -26,10 +26,10 @@ def UI(username, password):
                output_dir_input,
                logging_dir_input,
            ) = dreambooth_tab()
+        with gr.Tab('Dreambooth LoRA'):
+            lora_tab()
        with gr.Tab('Finetune'):
            finetune_tab()
-        with gr.Tab('LoRA'):
-            lora_tab()
        with gr.Tab('Utilities'):
            utilities_tab(
                train_data_dir_input=train_data_dir_input,
--- a/library/basic_caption_gui.py
+++ b/library/basic_caption_gui.py
@ -75,7 +75,7 @@ def gradio_basic_caption_gui_tab():
            )
        with gr.Row():
            prefix = gr.Textbox(
-                label='Prefix to add to txt caption',
+                label='Prefix to add to caption',
                placeholder='(Optional)',
                interactive=True,
            )
@ -85,7 +85,7 @@ def gradio_basic_caption_gui_tab():
                interactive=True,
            )
            postfix = gr.Textbox(
-                label='Postfix to add to txt caption',
+                label='Postfix to add to caption',
                placeholder='(Optional)',
                interactive=True,
            )
--- a/library/model_util.py
+++ b/library/model_util.py
--- a/lora_gui.py
+++ b/lora_gui.py
@ -42,7 +42,6 @@ def save_configuration(
    reg_data_dir,
    output_dir,
    max_resolution,
-    learning_rate,
    lr_scheduler,
    lr_warmup,
    train_batch_size,
@ -65,7 +64,7 @@ def save_configuration(
    shuffle_caption,
    save_state,
    resume,
-    prior_loss_weight, text_encoder_lr, unet_lr, network_train, network_dim
+    prior_loss_weight, text_encoder_lr, unet_lr, network_dim, lora_network_weights
 ):
    original_file_path = file_path

@ -94,7 +93,6 @@ def save_configuration(
        'reg_data_dir': reg_data_dir,
        'output_dir': output_dir,
        'max_resolution': max_resolution,
-        'learning_rate': learning_rate,
        'lr_scheduler': lr_scheduler,
        'lr_warmup': lr_warmup,
        'train_batch_size': train_batch_size,
@ -120,8 +118,8 @@ def save_configuration(
        'prior_loss_weight': prior_loss_weight,
        'text_encoder_lr': text_encoder_lr,
        'unet_lr': unet_lr,
-        'network_train': network_train,
-        'network_dim': network_dim
+        'network_dim': network_dim,
+        'lora_network_weights': lora_network_weights,
    }

    # Save the data to the selected file
@ -141,7 +139,6 @@ def open_configuration(
    reg_data_dir,
    output_dir,
    max_resolution,
-    learning_rate,
    lr_scheduler,
    lr_warmup,
    train_batch_size,
@ -164,7 +161,7 @@ def open_configuration(
    shuffle_caption,
    save_state,
    resume,
-    prior_loss_weight, text_encoder_lr, unet_lr, network_train, network_dim
+    prior_loss_weight, text_encoder_lr, unet_lr, network_dim, lora_network_weights
 ):

    original_file_path = file_path
@ -192,7 +189,6 @@ def open_configuration(
        my_data.get('reg_data_dir', reg_data_dir),
        my_data.get('output_dir', output_dir),
        my_data.get('max_resolution', max_resolution),
-        my_data.get('learning_rate', learning_rate),
        my_data.get('lr_scheduler', lr_scheduler),
        my_data.get('lr_warmup', lr_warmup),
        my_data.get('train_batch_size', train_batch_size),
@ -220,8 +216,8 @@ def open_configuration(
        my_data.get('prior_loss_weight', prior_loss_weight),
        my_data.get('text_encoder_lr', text_encoder_lr),
        my_data.get('unet_lr', unet_lr),
-        my_data.get('network_train', network_train),
        my_data.get('network_dim', network_dim),
+        my_data.get('lora_network_weights', lora_network_weights),
    )


@ -234,7 +230,6 @@ def train_model(
    reg_data_dir,
    output_dir,
    max_resolution,
-    learning_rate,
    lr_scheduler,
    lr_warmup,
    train_batch_size,
@ -257,7 +252,7 @@ def train_model(
    shuffle_caption,
    save_state,
    resume,
-    prior_loss_weight, text_encoder_lr, unet_lr, network_train, network_dim
+    prior_loss_weight, text_encoder_lr, unet_lr, network_dim, lora_network_weights
 ):
    def save_inference_file(output_dir, v2, v_parameterization):
        # Copy inference model for v2 if required
@ -294,6 +289,14 @@ def train_model(
    if output_dir == '':
        msgbox('Output folder path is missing')
        return
+    
+    # If string is empty set string to 0.
+    if text_encoder_lr == '': text_encoder_lr = 0
+    if unet_lr == '': unet_lr = 0
+    
+    if (float(text_encoder_lr) == 0) and (float(unet_lr) == 0):
+        msgbox('At least one Learning Rate value for "Text encoder" or "Unet" need to be provided')
+        return

    # Get a list of all subfolders in train_data_dir
    subfolders = [
@ -394,7 +397,7 @@ def train_model(
    run_cmd += f' --resolution={max_resolution}'
    run_cmd += f' --output_dir={output_dir}'
    run_cmd += f' --train_batch_size={train_batch_size}'
-    run_cmd += f' --learning_rate={learning_rate}'
+    # run_cmd += f' --learning_rate={learning_rate}'
    run_cmd += f' --lr_scheduler={lr_scheduler}'
    run_cmd += f' --lr_warmup_steps={lr_warmup_steps}'
    run_cmd += f' --max_train_steps={max_train_steps}'
@ -418,15 +421,20 @@ def train_model(
    if not float(prior_loss_weight) == 1.0:
        run_cmd += f' --prior_loss_weight={prior_loss_weight}'
    run_cmd += f' --network_module=networks.lora'
-    if not text_encoder_lr == '':
+    if not float(text_encoder_lr) == 0:
        run_cmd += f' --text_encoder_lr={text_encoder_lr}'
-    if not unet_lr == '':
-        run_cmd += f' --unet_lr={unet_lr}'
-    if network_train == 'Text encoder only':
-        run_cmd += f' --network_train_text_encoder_only'
-    elif network_train == 'Unet only':
+    else:
        run_cmd += f' --network_train_unet_only'
+    if not float(unet_lr) == 0:
+        run_cmd += f' --unet_lr={unet_lr}'
+    else:
+        run_cmd += f' --network_train_text_encoder_only'
+    # if network_train == 'Text encoder only':
+    #     run_cmd += f' --network_train_text_encoder_only'
+    # elif network_train == 'Unet only':
+    #     run_cmd += f' --network_train_unet_only'
    run_cmd += f' --network_dim={network_dim}'
+    run_cmd += f' --network_weights={lora_network_weights}'
    

    print(run_cmd)
@ -503,13 +511,13 @@ def UI(username, password):
    interface = gr.Blocks(css=css)

    with interface:
-        with gr.Tab('Dreambooth'):
+        with gr.Tab('LoRA'):
            (
                train_data_dir_input,
                reg_data_dir_input,
                output_dir_input,
                logging_dir_input,
-            ) = dreambooth_tab()
+            ) = lora_tab()
        with gr.Tab('Utilities'):
            utilities_tab(
                train_data_dir_input=train_data_dir_input,
@ -563,7 +571,7 @@ def lora_tab(
                document_symbol, elem_id='open_folder_small'
            )
            pretrained_model_name_or_path_file.click(
-                get_file_path,
+                get_any_file_path,
                inputs=[pretrained_model_name_or_path_input],
                outputs=pretrained_model_name_or_path_input,
            )
@ -597,19 +605,7 @@ def lora_tab(
                ],
                value='same as source model',
            )
-        with gr.Row():
-            lora_network_weights = gr.Textbox(
-                label='LoRA network weights',
-                placeholder='{Optional) Path to existing LoRA network weights to resume training}',
-            )
-            lora_network_weights_file = gr.Button(
-                document_symbol, elem_id='open_folder_small'
-            )
-            lora_network_weights_file.click(
-                get_any_file_path,
-                inputs=[lora_network_weights],
-                outputs=lora_network_weights,
-            )
+            
        with gr.Row():
            v2_input = gr.Checkbox(label='v2', value=True)
            v_parameterization_input = gr.Checkbox(
@ -695,7 +691,20 @@ def lora_tab(
        )
    with gr.Tab('Training parameters'):
        with gr.Row():
-            learning_rate_input = gr.Textbox(label='Learning rate', value=1e-4)
+            lora_network_weights = gr.Textbox(
+                label='LoRA network weights',
+                placeholder='{Optional) Path to existing LoRA network weights to resume training',
+            )
+            lora_network_weights_file = gr.Button(
+                document_symbol, elem_id='open_folder_small'
+            )
+            lora_network_weights_file.click(
+                get_any_file_path,
+                inputs=[lora_network_weights],
+                outputs=lora_network_weights,
+            )
+        with gr.Row():
+            # learning_rate_input = gr.Textbox(label='Learning rate', value=1e-4, visible=False)
            lr_scheduler_input = gr.Dropdown(
                label='LR Scheduler',
                choices=[
@ -712,16 +721,16 @@ def lora_tab(
        with gr.Row():
            text_encoder_lr = gr.Textbox(label='Text Encoder learning rate', value=1e-6, placeholder='Optional')
            unet_lr = gr.Textbox(label='Unet learning rate', value=1e-4, placeholder='Optional')
-            network_train =gr.Dropdown(
-                label='Network to train',
-                choices=[
-                    'Text encoder and Unet',
-                    'Text encoder only',
-                    'Unet only',
-                ],
-                value='Text encoder and Unet',
-                interactive=True
-            )
+            # network_train = gr.Dropdown(
+            #     label='Network to train',
+            #     choices=[
+            #         'Text encoder and Unet',
+            #         'Text encoder only',
+            #         'Unet only',
+            #     ],
+            #     value='Text encoder and Unet',
+            #     interactive=True
+            # )
            network_dim = gr.Slider(
                minimum=1,
                maximum=32,
@ -846,7 +855,7 @@ def lora_tab(
        reg_data_dir_input,
        output_dir_input,
        max_resolution_input,
-        learning_rate_input,
+        # learning_rate_input,
        lr_scheduler_input,
        lr_warmup_input,
        train_batch_size_input,
@ -869,7 +878,7 @@ def lora_tab(
        shuffle_caption,
        save_state,
        resume,
-        prior_loss_weight, text_encoder_lr, unet_lr, network_train, network_dim
+        prior_loss_weight, text_encoder_lr, unet_lr, network_dim, lora_network_weights
    ]

    button_open_config.click(
--- a/train_network.py
+++ b/train_network.py
@ -925,11 +925,12 @@ def train(args):
    print(f"update token length: {args.max_token_length}")

  # 学習データを用意する
+  assert args.resolution is not None, f"resolution is required / resolution（解像度）を指定してください"
  resolution = tuple([int(r) for r in args.resolution.split(',')])
  if len(resolution) == 1:
    resolution = (resolution[0], resolution[0])
  assert len(resolution) == 2, \
-      f"resolution must be 'size' or 'width,height' / resolutionは'サイズ'または'幅','高さ'で指定してください: {args.resolution}"
+      f"resolution must be 'size' or 'width,height' / resolution（解像度）は'サイズ'または'幅','高さ'で指定してください: {args.resolution}"

  if args.face_crop_aug_range is not None:
    face_crop_aug_range = tuple([float(r) for r in args.face_crop_aug_range.split(',')])
@ -1373,9 +1374,9 @@ if __name__ == '__main__':
                      help="keep heading N tokens when shuffling caption tokens / captionのシャッフル時に、先頭からこの個数のトークンをシャッフルしないで残す")
  parser.add_argument("--train_data_dir", type=str, default=None, help="directory for train images / 学習画像データのディレクトリ")
  parser.add_argument("--reg_data_dir", type=str, default=None, help="directory for regularization images / 正則化画像データのディレクトリ")
-  parser.add_argument("--in_json", type=str, default=None, help="json meatadata for dataset / データセットのmetadataのjsonファイル")
+  parser.add_argument("--in_json", type=str, default=None, help="json metadata for dataset / データセットのmetadataのjsonファイル")
  parser.add_argument("--caption_extension", type=str, default=".caption", help="extension of caption files / 読み込むcaptionファイルの拡張子")
-  parser.add_argument("--dataset_repeats", type=int, default=None,
+  parser.add_argument("--dataset_repeats", type=int, default=1,
                      help="repeat dataset when training with captions / キャプションでの学習時にデータセットを繰り返す回数")
  parser.add_argument("--output_dir", type=str, default=None,
                      help="directory to output trained model / 学習後のモデル出力先ディレクトリ")
@ -1449,4 +1450,4 @@ if __name__ == '__main__':
                      help="only training Text Encoder part / Text Encoder関連部分のみ学習する")

  args = parser.parse_args()
-  train(args)
+  train(args)