From b1fb87a9e17dfb760b99dfd0143bb25097d54b7c Mon Sep 17 00:00:00 2001 From: bmaltais Date: Tue, 28 Feb 2023 07:45:42 -0500 Subject: [PATCH 1/7] Merging PR into LoCon branch --- lora_gui.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/lora_gui.py b/lora_gui.py index eb0b567..e79d067 100644 --- a/lora_gui.py +++ b/lora_gui.py @@ -10,6 +10,7 @@ import os import subprocess import pathlib import argparse +import shutil from library.common_gui import ( get_folder_path, remove_doublequote, @@ -43,6 +44,12 @@ folder_symbol = '\U0001f4c2' # 📂 refresh_symbol = '\U0001f504' # 🔄 save_style_symbol = '\U0001f4be' # 💾 document_symbol = '\U0001F4C4' # 📄 +locon_path = os.getcwd()+'\\locon\\' + +def getlocon(): + os.system('git clone https://github.com/KohakuBlueleaf/LoCon.git') + os.system('ren '+locon_path[:-6]+'\\LoCon\\'+' locon_github-sourcecode') + shutil.copytree(locon_path[:-6]+'locon_github-sourcecode\\locon\\', locon_path) def save_configuration( save_as, @@ -104,7 +111,7 @@ def save_configuration( bucket_reso_steps, caption_dropout_every_n_epochs, caption_dropout_rate, optimizer, - optimizer_args,noise_offset, + optimizer_args,noise_offset, locon = 0 ): # Get list of function parameters and values parameters = list(locals().items()) @@ -210,7 +217,7 @@ def open_configuration( bucket_reso_steps, caption_dropout_every_n_epochs, caption_dropout_rate, optimizer, - optimizer_args,noise_offset, + optimizer_args,noise_offset, locon=0 ): # Get list of function parameters and values parameters = list(locals().items()) @@ -294,7 +301,7 @@ def train_model( bucket_reso_steps, caption_dropout_every_n_epochs, caption_dropout_rate, optimizer, - optimizer_args,noise_offset, + optimizer_args,noise_offset, locon ): if pretrained_model_name_or_path == '': msgbox('Source model information is missing') @@ -430,7 +437,12 @@ def train_model( run_cmd += f' --save_model_as={save_model_as}' if not float(prior_loss_weight) == 1.0: run_cmd += f' --prior_loss_weight={prior_loss_weight}' - run_cmd += f' --network_module=networks.lora' + if locon: + if not os.path.exists(locon_path): + getlocon() + run_cmd += ' --network_module=locon.locon_kohya' + else: + run_cmd += f' --network_module=networks.lora' if not (float(text_encoder_lr) == 0) or not (float(unet_lr) == 0): if not (float(text_encoder_lr) == 0) and not (float(unet_lr) == 0): @@ -678,6 +690,8 @@ def lora_tab( ) enable_bucket = gr.Checkbox(label='Enable buckets', value=True) with gr.Accordion('Advanced Configuration', open=False): + with gr.Row(): + locon= gr.Checkbox(label='Train a LoCon instead of a general LoRA (may not be able to merge now)', value=False) with gr.Row(): no_token_padding = gr.Checkbox( label='No token padding', value=False @@ -819,7 +833,7 @@ def lora_tab( bucket_reso_steps, caption_dropout_every_n_epochs, caption_dropout_rate, optimizer, - optimizer_args,noise_offset, + optimizer_args,noise_offset,locon ] button_open_config.click( From d76fe7d4e0aae696a992b58418756facb4e4c6b1 Mon Sep 17 00:00:00 2001 From: Ki-wimon <40882134+Ki-wimon@users.noreply.github.com> Date: Tue, 28 Feb 2023 22:58:35 +0800 Subject: [PATCH 2/7] LoCon script auto upgrade feature --- lora_gui.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/lora_gui.py b/lora_gui.py index 99c6609..3e1d163 100644 --- a/lora_gui.py +++ b/lora_gui.py @@ -10,7 +10,6 @@ import os import subprocess import pathlib import argparse -import shutil from library.common_gui import ( get_folder_path, remove_doublequote, @@ -44,12 +43,18 @@ folder_symbol = '\U0001f4c2' # 📂 refresh_symbol = '\U0001f504' # 🔄 save_style_symbol = '\U0001f4be' # 💾 document_symbol = '\U0001F4C4' # 📄 -locon_path = os.getcwd()+'\\locon\\' +path_of_this_folder = os.getcwd() -def getlocon(): - os.system('git clone https://github.com/KohakuBlueleaf/LoCon.git') - os.system('ren '+locon_path[:-6]+'\\LoCon\\'+' locon_github-sourcecode') - shutil.copytree(locon_path[:-6]+'locon_github-sourcecode\\locon\\', locon_path) +def getlocon(existance): + if existance: + print('Checking LoCon script version...') + now_path = os.getcwd() + os.chdir(os.path.join(path_of_this_folder, 'locon')) + os.system('git pull') + os.chdir(now_path) + else: + os.system('git clone https://github.com/KohakuBlueleaf/LoCon.git') + os.system('ren '+ os.path.join(path_of_this_folder, 'LoCon') +' locon') def save_configuration( save_as, @@ -438,9 +443,8 @@ def train_model( if not float(prior_loss_weight) == 1.0: run_cmd += f' --prior_loss_weight={prior_loss_weight}' if locon: - if not os.path.exists(locon_path): - getlocon() - run_cmd += ' --network_module=locon.locon_kohya' + getlocon(os.path.exists(os.path.join(path_of_this_folder, 'locon'))) + run_cmd += f' --network_module=locon.locon.locon_kohya' else: run_cmd += f' --network_module=networks.lora' From c07e3bba76f707c4863d465ebd104aafab5e2f4c Mon Sep 17 00:00:00 2001 From: Ki-wimon <40882134+Ki-wimon@users.noreply.github.com> Date: Wed, 1 Mar 2023 12:19:18 +0800 Subject: [PATCH 3/7] add new LoCon args --- lora_gui.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/lora_gui.py b/lora_gui.py index 3e1d163..79c17b7 100644 --- a/lora_gui.py +++ b/lora_gui.py @@ -445,6 +445,7 @@ def train_model( if locon: getlocon(os.path.exists(os.path.join(path_of_this_folder, 'locon'))) run_cmd += f' --network_module=locon.locon.locon_kohya' + run_cmd += f' --network_args "conv_dim={conv_dim}" "conv_alpha={conv_alpha}"' else: run_cmd += f' --network_module=networks.lora' @@ -664,19 +665,19 @@ def lora_tab( placeholder='Optional', ) network_dim = gr.Slider( - minimum=4, + minimum=1, maximum=1024, label='Network Rank (Dimension)', value=8, - step=4, + step=1, interactive=True, ) network_alpha = gr.Slider( - minimum=4, + minimum=1, maximum=1024, label='Network Alpha', value=1, - step=4, + step=1, interactive=True, ) with gr.Row(): @@ -696,6 +697,20 @@ def lora_tab( with gr.Accordion('Advanced Configuration', open=False): with gr.Row(): locon= gr.Checkbox(label='Train a LoCon instead of a general LoRA (may not be able to merge now)', value=False) + conv_dim = gr.Slider( + minimum=1, + maximum=512, + value=0, + step=1, + label='LoCon Convolution Rank (Dimension)', + ) + conv_alpha = gr.Slider( + minimum=1, + maximum=512, + value=0, + step=1, + label='LoCon Convolution Alpha', + ) with gr.Row(): no_token_padding = gr.Checkbox( label='No token padding', value=False From 6bcd52c9cc6aefd783a8dd0baa2b218343d68694 Mon Sep 17 00:00:00 2001 From: Ki-wimon <40882134+Ki-wimon@users.noreply.github.com> Date: Wed, 1 Mar 2023 12:22:11 +0800 Subject: [PATCH 4/7] update new locon args support --- lora_gui.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lora_gui.py b/lora_gui.py index 79c17b7..6cf856e 100644 --- a/lora_gui.py +++ b/lora_gui.py @@ -116,7 +116,8 @@ def save_configuration( bucket_reso_steps, caption_dropout_every_n_epochs, caption_dropout_rate, optimizer, - optimizer_args,noise_offset, locon = 0 + optimizer_args,noise_offset, + locon=0, conv_dim=0, conv_alpha=0, ): # Get list of function parameters and values parameters = list(locals().items()) @@ -222,7 +223,8 @@ def open_configuration( bucket_reso_steps, caption_dropout_every_n_epochs, caption_dropout_rate, optimizer, - optimizer_args,noise_offset, locon=0 + optimizer_args,noise_offset, + locon=0, conv_dim=0, conv_alpha=0, ): # Get list of function parameters and values parameters = list(locals().items()) @@ -306,7 +308,8 @@ def train_model( bucket_reso_steps, caption_dropout_every_n_epochs, caption_dropout_rate, optimizer, - optimizer_args,noise_offset, locon + optimizer_args,noise_offset, + locon, conv_dim, conv_alpha, ): if pretrained_model_name_or_path == '': msgbox('Source model information is missing') @@ -852,7 +855,8 @@ def lora_tab( bucket_reso_steps, caption_dropout_every_n_epochs, caption_dropout_rate, optimizer, - optimizer_args,noise_offset,locon + optimizer_args,noise_offset, + locon, conv_dim, conv_alpha, ] button_open_config.click( From cdb8bb1182d2241872387b67c81a7f8748d5e90e Mon Sep 17 00:00:00 2001 From: Ki-wimon <40882134+Ki-wimon@users.noreply.github.com> Date: Wed, 1 Mar 2023 13:07:38 +0800 Subject: [PATCH 5/7] update code of cloning locon scripts --- lora_gui.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lora_gui.py b/lora_gui.py index 6cf856e..ac3fc11 100644 --- a/lora_gui.py +++ b/lora_gui.py @@ -46,15 +46,16 @@ document_symbol = '\U0001F4C4' # 📄 path_of_this_folder = os.getcwd() def getlocon(existance): + now_path = os.getcwd() if existance: print('Checking LoCon script version...') - now_path = os.getcwd() os.chdir(os.path.join(path_of_this_folder, 'locon')) os.system('git pull') os.chdir(now_path) else: - os.system('git clone https://github.com/KohakuBlueleaf/LoCon.git') - os.system('ren '+ os.path.join(path_of_this_folder, 'LoCon') +' locon') + os.chdir(path_of_this_folder) + os.system('git clone https://github.com/KohakuBlueleaf/LoCon.git locon') + os.chdir(now_path) def save_configuration( save_as, From 88a49df47e8e9a2b13d38c33f1d0cd5e10863e4b Mon Sep 17 00:00:00 2001 From: Ki-wimon <40882134+Ki-wimon@users.noreply.github.com> Date: Wed, 1 Mar 2023 18:57:02 +0800 Subject: [PATCH 6/7] set inital value to LoCon conv parameters --- lora_gui.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lora_gui.py b/lora_gui.py index ac3fc11..a29dae6 100644 --- a/lora_gui.py +++ b/lora_gui.py @@ -704,14 +704,14 @@ def lora_tab( conv_dim = gr.Slider( minimum=1, maximum=512, - value=0, + value=1, step=1, label='LoCon Convolution Rank (Dimension)', ) conv_alpha = gr.Slider( minimum=1, maximum=512, - value=0, + value=1, step=1, label='LoCon Convolution Alpha', ) From 5bfd478d32b61bc9a71d955a6a870eab63900a2d Mon Sep 17 00:00:00 2001 From: Ki-wimon <40882134+Ki-wimon@users.noreply.github.com> Date: Wed, 1 Mar 2023 20:27:23 +0800 Subject: [PATCH 7/7] update description to LoCon training --- lora_gui.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lora_gui.py b/lora_gui.py index a29dae6..aab56e8 100644 --- a/lora_gui.py +++ b/lora_gui.py @@ -700,7 +700,7 @@ def lora_tab( enable_bucket = gr.Checkbox(label='Enable buckets', value=True) with gr.Accordion('Advanced Configuration', open=False): with gr.Row(): - locon= gr.Checkbox(label='Train a LoCon instead of a general LoRA (may not be able to merge now)', value=False) + locon= gr.Checkbox(label='Train a LoCon instead of a general LoRA (does not support v2 base models) (may not be able to some utilities now)', value=False) conv_dim = gr.Slider( minimum=1, maximum=512,