195 lines
6.9 KiB
Python
195 lines
6.9 KiB
Python
# extract approximating LoRA by svd from two SD models
|
||
# The code is based on https://github.com/cloneofsimo/lora/blob/develop/lora_diffusion/cli_svd.py
|
||
# Thanks to cloneofsimo!
|
||
|
||
import argparse
|
||
import os
|
||
import torch
|
||
from safetensors.torch import load_file, save_file
|
||
from tqdm import tqdm
|
||
import library.model_util as model_util
|
||
import lora
|
||
import numpy as np
|
||
|
||
|
||
CLAMP_QUANTILE = 1 # 0.99
|
||
MIN_DIFF = 1e-6
|
||
|
||
|
||
def save_to_file(file_name, model, state_dict, dtype):
|
||
if dtype is not None:
|
||
for key in list(state_dict.keys()):
|
||
if type(state_dict[key]) == torch.Tensor:
|
||
state_dict[key] = state_dict[key].to(dtype)
|
||
|
||
if os.path.splitext(file_name)[1] == '.safetensors':
|
||
save_file(model, file_name)
|
||
else:
|
||
torch.save(model, file_name)
|
||
|
||
|
||
def svd(args):
|
||
def str_to_dtype(p):
|
||
if p == 'float':
|
||
return torch.float
|
||
if p == 'fp16':
|
||
return torch.float16
|
||
if p == 'bf16':
|
||
return torch.bfloat16
|
||
return None
|
||
|
||
save_dtype = str_to_dtype(args.save_precision)
|
||
|
||
print(f"loading SD model : {args.model_org}")
|
||
text_encoder_o, _, unet_o = model_util.load_models_from_stable_diffusion_checkpoint(args.v2, args.model_org)
|
||
print(f"loading SD model : {args.model_tuned}")
|
||
text_encoder_t, _, unet_t = model_util.load_models_from_stable_diffusion_checkpoint(args.v2, args.model_tuned)
|
||
|
||
# create LoRA network to extract weights: Use dim (rank) as alpha
|
||
lora_network_o = lora.create_network(1.0, args.dim, args.dim * 1.5, None, text_encoder_o, unet_o)
|
||
lora_network_t = lora.create_network(1.0, args.dim, args.dim * 1.5, None, text_encoder_t, unet_t)
|
||
assert len(lora_network_o.text_encoder_loras) == len(
|
||
lora_network_t.text_encoder_loras), f"model version is different (SD1.x vs SD2.x) / それぞれのモデルのバージョンが違います（SD1.xベースとSD2.xベース） "
|
||
|
||
# get diffs
|
||
diffs = {}
|
||
text_encoder_different = False
|
||
for i, (lora_o, lora_t) in enumerate(zip(lora_network_o.text_encoder_loras, lora_network_t.text_encoder_loras)):
|
||
lora_name = lora_o.lora_name
|
||
module_o = lora_o.org_module
|
||
module_t = lora_t.org_module
|
||
diff = module_t.weight - module_o.weight
|
||
|
||
# Text Encoder might be same
|
||
if torch.max(torch.abs(diff)) > MIN_DIFF:
|
||
text_encoder_different = True
|
||
|
||
diff = diff.float()
|
||
diffs[lora_name] = diff
|
||
|
||
if not text_encoder_different:
|
||
print("Text encoder is same. Extract U-Net only.")
|
||
lora_network_o.text_encoder_loras = []
|
||
diffs = {}
|
||
|
||
for i, (lora_o, lora_t) in enumerate(zip(lora_network_o.unet_loras, lora_network_t.unet_loras)):
|
||
lora_name = lora_o.lora_name
|
||
module_o = lora_o.org_module
|
||
module_t = lora_t.org_module
|
||
diff = module_t.weight - module_o.weight
|
||
diff = diff.float()
|
||
|
||
if args.device:
|
||
diff = diff.to(args.device)
|
||
|
||
diffs[lora_name] = diff
|
||
|
||
# make LoRA with SVD
|
||
print("calculating by SVD")
|
||
rank = args.dim
|
||
lora_weights = {}
|
||
with torch.no_grad():
|
||
for lora_name, mat in tqdm(list(diffs.items())):
|
||
conv2d = (len(mat.size()) == 4)
|
||
if conv2d:
|
||
mat = mat.squeeze()
|
||
|
||
U, S, Vt = torch.linalg.svd(mat)
|
||
|
||
U = U[:, :rank]
|
||
S = S[:rank]
|
||
U = U @ torch.diag(S)
|
||
|
||
Vt = Vt[:rank, :]
|
||
|
||
lora_weights[lora_name] = (U, Vt)
|
||
|
||
# # make LoRA with svd
|
||
# print("calculating by svd")
|
||
# rank = args.dim
|
||
# lora_weights = {}
|
||
# with torch.no_grad():
|
||
# for lora_name, mat in tqdm(list(diffs.items())):
|
||
# conv2d = (len(mat.size()) == 4)
|
||
# if conv2d:
|
||
# mat = mat.squeeze()
|
||
|
||
# U, S, Vh = torch.linalg.svd(mat)
|
||
|
||
# U = U[:, :rank]
|
||
# S = S[:rank]
|
||
# U = U @ torch.diag(S)
|
||
|
||
# Vh = Vh[:rank, :]
|
||
|
||
# # create new tensors directly from the numpy arrays
|
||
# U = torch.as_tensor(U)
|
||
# Vh = torch.as_tensor(Vh)
|
||
|
||
# # dist = torch.cat([U.flatten(), Vh.flatten()])
|
||
# # hi_val = torch.quantile(dist, CLAMP_QUANTILE)
|
||
# # low_val = -hi_val
|
||
|
||
# # U = U.clamp(low_val, hi_val)
|
||
# # Vh = Vh.clamp(low_val, hi_val)
|
||
|
||
# # # soft thresholding
|
||
# # alpha = S[-1] / 1000.0 # adjust this parameter as needed
|
||
# # U = torch.sign(U) * torch.nn.functional.relu(torch.abs(U) - alpha)
|
||
# # Vh = torch.sign(Vh) * torch.nn.functional.relu(torch.abs(Vh) - alpha)
|
||
|
||
# lora_weights[lora_name] = (U, Vh)
|
||
|
||
# make state dict for LoRA
|
||
lora_network_o.apply_to(text_encoder_o, unet_o, text_encoder_different, True) # to make state dict
|
||
lora_sd = lora_network_o.state_dict()
|
||
print(f"LoRA has {len(lora_sd)} weights.")
|
||
|
||
for key in list(lora_sd.keys()):
|
||
if "alpha" in key:
|
||
continue
|
||
|
||
lora_name = key.split('.')[0]
|
||
i = 0 if "lora_up" in key else 1
|
||
|
||
weights = lora_weights[lora_name][i]
|
||
# print(key, i, weights.size(), lora_sd[key].size())
|
||
if len(lora_sd[key].size()) == 4:
|
||
weights = weights.unsqueeze(2).unsqueeze(3)
|
||
|
||
assert weights.size() == lora_sd[key].size(), f"size unmatch: {key}"
|
||
lora_sd[key] = weights
|
||
|
||
# load state dict to LoRA and save it
|
||
info = lora_network_o.load_state_dict(lora_sd)
|
||
print(f"Loading extracted LoRA weights: {info}")
|
||
|
||
dir_name = os.path.dirname(args.save_to)
|
||
if dir_name and not os.path.exists(dir_name):
|
||
os.makedirs(dir_name, exist_ok=True)
|
||
|
||
# minimum metadata
|
||
metadata = {"ss_network_dim": str(args.dim), "ss_network_alpha": str(args.dim * 1.5)}
|
||
|
||
lora_network_o.save_weights(args.save_to, save_dtype, metadata)
|
||
print(f"LoRA weights are saved to: {args.save_to}")
|
||
|
||
|
||
if __name__ == '__main__':
|
||
parser = argparse.ArgumentParser()
|
||
parser.add_argument("--v2", action='store_true',
|
||
help='load Stable Diffusion v2.x model / Stable Diffusion 2.xのモデルを読み込む')
|
||
parser.add_argument("--save_precision", type=str, default=None,
|
||
choices=[None, "float", "fp16", "bf16"], help="precision in saving, same to merging if omitted / 保存時に精度を変更して保存する、省略時はfloat")
|
||
parser.add_argument("--model_org", type=str, default=None,
|
||
help="Stable Diffusion original model: ckpt or safetensors file / 元モデル、ckptまたはsafetensors")
|
||
parser.add_argument("--model_tuned", type=str, default=None,
|
||
help="Stable Diffusion tuned model, LoRA is difference of `original to tuned`: ckpt or safetensors file / 派生モデル（生成されるLoRAは元→派生の差分になります）、ckptまたはsafetensors")
|
||
parser.add_argument("--save_to", type=str, default=None,
|
||
help="destination file name: ckpt or safetensors file / 保存先のファイル名、ckptまたはsafetensors")
|
||
parser.add_argument("--dim", type=int, default=4, help="dimension (rank) of LoRA (default 4) / LoRAの次元数（rank）（デフォルト4）")
|
||
parser.add_argument("--device", type=str, default=None, help="device to use, cuda for GPU / 計算を行うデバイス、cuda でGPUを使う")
|
||
|
||
args = parser.parse_args()
|
||
svd(args)
|