LoHa support

README.md

@@ -177,6 +177,7 @@ This will store your a backup file with your current locally installed pip packa
 ## Change History
 
 * 2023/03/05 (v21.2.0):
+    - Add support for LoRA LoHa type. See https://github.com/KohakuBlueleaf/LyCORIS for more detais.
     - Fix issue https://github.com/bmaltais/kohya_ss/issues/335
     - Add option to print LoRA trainer command without executing it
     - Add support for samples during trainin via a new `Sample images config` accordion in the `Training parameters` tab.

library/common_gui.py

@@ -59,6 +59,9 @@ def update_my_data(my_data):
             else:
                 my_data[key] = -1
                 
+    if my_data.get('LoRA_type', 'Standard') == 'LoCon':
+        my_data['LoRA_type'] = 'LyCORIS/LoCon'
+        
     return my_data
 
 

lora_gui.py

@@ -474,17 +474,29 @@ def train_model(
         run_cmd += f' --save_model_as={save_model_as}'
     if not float(prior_loss_weight) == 1.0:
         run_cmd += f' --prior_loss_weight={prior_loss_weight}'
-    if LoRA_type == 'LoCon':
+    if LoRA_type == 'LoCon' or LoRA_type == 'LyCORIS/LoCon':
         try:
-            import locon.locon_kohya
+            import lycoris
         except ModuleNotFoundError:
             print(
-                "\033[1;31mError:\033[0m The required module 'locon' is not installed. Please install by running \033[33mupgrade.ps1\033[0m before running this program."
+                "\033[1;31mError:\033[0m The required module 'lycoris_lora' is not installed. Please install by running \033[33mupgrade.ps1\033[0m before running this program."
             )
             return
-        run_cmd += f' --network_module=locon.locon_kohya'
+        run_cmd += f' --network_module=lycoris.kohya'
         run_cmd += (
-            f' --network_args "conv_dim={conv_dim}" "conv_alpha={conv_alpha}"'
+            f' --network_args "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "algo=lora"'
+        )
+    if LoRA_type == 'LyCORIS/LoHa':
+        try:
+            import lycoris
+        except ModuleNotFoundError:
+            print(
+                "\033[1;31mError:\033[0m The required module 'lycoris_lora' is not installed. Please install by running \033[33mupgrade.ps1\033[0m before running this program."
+            )
+            return
+        run_cmd += f' --network_module=lycoris.kohya'
+        run_cmd += (
+            f' --network_args "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "algo=loha"'
         )
     if LoRA_type == 'Kohya LoCon':
         run_cmd += f' --network_module=networks.lora'
@@ -700,7 +712,9 @@ def lora_tab(
                 label='LoRA type',
                 choices=[
                     'Kohya LoCon',
-                    'LoCon',
+                    # 'LoCon',
+                    'LyCORIS/LoCon',
+                    'LyCORIS/LoHa',
                     'Standard',
                 ],
                 value='Standard',
@@ -787,7 +801,7 @@ def lora_tab(
         # Show of hide LoCon conv settings depending on LoRA type selection
         def LoRA_type_change(LoRA_type):
             print('LoRA type changed...')
-            if LoRA_type == 'LoCon' or LoRA_type == 'Kohya LoCon':
+            if LoRA_type == 'LoCon' or LoRA_type == 'Kohya LoCon' or LoRA_type == 'LyCORIS/LoHa' or LoRA_type == 'LyCORIS/LoCon':
                 return gr.Group.update(visible=True)
             else:
                 return gr.Group.update(visible=False)

requirements.txt

@@ -25,6 +25,6 @@ timm==0.6.12
 huggingface-hub==0.12.0
 tensorflow==2.10.1
 # For locon support
-locon==0.0.2
+lycoris_lora==0.0.5
 # for kohya_ss library
 .