2023-03-27 19:37:00 +00:00
|
|
|
#!/usr/bin/env bash
|
|
|
|
|
2023-03-28 01:58:03 +00:00
|
|
|
# This file will be the host environment setup file for all operating systems other than base Windows.
|
|
|
|
|
2023-04-02 19:32:56 +00:00
|
|
|
# Set the required package versions here.
|
|
|
|
# They will be appended to the requirements.txt file in the installation directory.
|
|
|
|
TENSORFLOW_VERSION="2.12.0"
|
|
|
|
TENSORFLOW_MACOS_VERSION="2.12.0"
|
|
|
|
TENSORFLOW_METAL_VERSION="0.8.0"
|
|
|
|
|
2023-03-29 01:07:18 +00:00
|
|
|
display_help() {
|
|
|
|
cat <<EOF
|
|
|
|
Kohya_SS Installation Script for POSIX operating systems.
|
|
|
|
|
|
|
|
Usage:
|
2023-03-31 01:20:42 +00:00
|
|
|
# Specifies custom branch, install directory, and git repo
|
2023-03-29 07:47:03 +00:00
|
|
|
setup.sh -b dev -d /workspace/kohya_ss -g https://mycustom.repo.tld/custom_fork.git
|
2023-03-31 01:20:42 +00:00
|
|
|
|
|
|
|
# Same as example 1, but uses long options
|
2023-03-29 07:47:03 +00:00
|
|
|
setup.sh --branch=dev --dir=/workspace/kohya_ss --git-repo=https://mycustom.repo.tld/custom_fork.git
|
2023-03-29 01:07:18 +00:00
|
|
|
|
2023-03-31 23:39:28 +00:00
|
|
|
# Maximum verbosity, fully automated installation in a runpod environment skipping the runpod env checks
|
2023-03-31 01:20:42 +00:00
|
|
|
setup.sh -vvv --skip-space-check --runpod
|
|
|
|
|
2023-03-29 01:07:18 +00:00
|
|
|
Options:
|
2023-03-31 23:39:28 +00:00
|
|
|
-b BRANCH, --branch=BRANCH Select which branch of kohya to check out on new installs.
|
2023-03-29 01:07:18 +00:00
|
|
|
-d DIR, --dir=DIR The full path you want kohya_ss installed to.
|
2023-04-01 04:38:27 +00:00
|
|
|
-g REPO, --git_repo=REPO You can optionally provide a git repo to check out for runpod installation. Useful for custom forks.
|
2023-03-31 01:20:42 +00:00
|
|
|
-h, --help Show this screen.
|
|
|
|
-i, --interactive Interactively configure accelerate instead of using default config file.
|
2023-04-01 04:38:27 +00:00
|
|
|
-n, --no-git-update Do not update kohya_ss repo. No git pull or clone operations.
|
2023-03-29 21:41:17 +00:00
|
|
|
-p, --public Expose public URL in runpod mode. Won't have an effect in other modes.
|
2023-03-29 07:47:03 +00:00
|
|
|
-r, --runpod Forces a runpod installation. Useful if detection fails for any reason.
|
2023-03-31 01:20:42 +00:00
|
|
|
-s, --skip-space-check Skip the 10Gb minimum storage space check.
|
|
|
|
-v, --verbose Increase verbosity levels up to 3.
|
2023-03-29 01:07:18 +00:00
|
|
|
EOF
|
|
|
|
}
|
|
|
|
|
2023-03-29 23:05:57 +00:00
|
|
|
# Checks to see if variable is set and non-empty.
|
|
|
|
# This is defined first, so we can use the function for some default variable values
|
|
|
|
env_var_exists() {
|
2023-04-01 20:28:05 +00:00
|
|
|
if [[ -n "${!1}" ]]; then
|
2023-04-01 20:22:30 +00:00
|
|
|
return 0
|
2023-04-01 20:28:05 +00:00
|
|
|
else
|
|
|
|
return 1
|
2023-03-29 23:05:57 +00:00
|
|
|
fi
|
|
|
|
}
|
|
|
|
|
|
|
|
# Need RUNPOD to have a default value before first access
|
|
|
|
RUNPOD=false
|
|
|
|
if env_var_exists RUNPOD_POD_ID || env_var_exists RUNPOD_API_KEY; then
|
|
|
|
RUNPOD=true
|
|
|
|
fi
|
|
|
|
|
2023-04-01 02:30:43 +00:00
|
|
|
# This gets the directory the script is run from so pathing can work relative to the script where needed.
|
2023-03-31 23:39:28 +00:00
|
|
|
SCRIPT_DIR="$(cd -- $(dirname -- "$0") && pwd)"
|
|
|
|
|
2023-03-29 01:07:18 +00:00
|
|
|
# Variables defined before the getopts loop, so we have sane default values.
|
2023-03-29 23:05:57 +00:00
|
|
|
# Default installation locations based on OS and environment
|
|
|
|
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
|
|
|
|
if [ "$RUNPOD" = true ]; then
|
|
|
|
DIR="/workspace/kohya_ss"
|
2023-03-31 23:39:28 +00:00
|
|
|
elif [ -d "$SCRIPT_DIR/.git" ]; then
|
|
|
|
DIR="$SCRIPT_DIR"
|
2023-03-29 23:05:57 +00:00
|
|
|
elif [ -w "/opt" ]; then
|
|
|
|
DIR="/opt/kohya_ss"
|
|
|
|
elif env_var_exists HOME; then
|
|
|
|
DIR="$HOME/kohya_ss"
|
|
|
|
else
|
|
|
|
# The last fallback is simply PWD
|
|
|
|
DIR="$(PWD)"
|
|
|
|
fi
|
|
|
|
else
|
2023-03-31 23:39:28 +00:00
|
|
|
if [ -d "$SCRIPT_DIR/.git" ]; then
|
|
|
|
DIR="$SCRIPT_DIR"
|
|
|
|
elif env_var_exists HOME; then
|
2023-03-29 23:05:57 +00:00
|
|
|
DIR="$HOME/kohya_ss"
|
|
|
|
else
|
|
|
|
# The last fallback is simply PWD
|
|
|
|
DIR="$(PWD)"
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
2023-03-31 01:20:42 +00:00
|
|
|
VERBOSITY=2 #Start counting at 2 so that any increase to this will result in a minimum of file descriptor 3. You should leave this alone.
|
|
|
|
MAXVERBOSITY=6 #The highest verbosity we use / allow to be displayed. Feel free to adjust.
|
|
|
|
|
2023-03-29 17:53:15 +00:00
|
|
|
BRANCH="master"
|
2023-03-29 07:47:03 +00:00
|
|
|
GIT_REPO="https://github.com/bmaltais/kohya_ss.git"
|
2023-03-29 20:26:16 +00:00
|
|
|
INTERACTIVE=false
|
2023-03-29 21:41:17 +00:00
|
|
|
PUBLIC=false
|
2023-03-31 01:20:42 +00:00
|
|
|
SKIP_SPACE_CHECK=false
|
2023-03-31 23:39:28 +00:00
|
|
|
SKIP_GIT_UPDATE=false
|
2023-03-29 01:07:18 +00:00
|
|
|
|
2023-03-31 23:39:28 +00:00
|
|
|
while getopts ":vb:d:g:inprs-:" opt; do
|
2023-03-29 01:07:18 +00:00
|
|
|
# support long options: https://stackoverflow.com/a/28466267/519360
|
|
|
|
if [ "$opt" = "-" ]; then # long option: reformulate OPT and OPTARG
|
|
|
|
opt="${OPTARG%%=*}" # extract long option name
|
|
|
|
OPTARG="${OPTARG#$opt}" # extract long option argument (may be empty)
|
|
|
|
OPTARG="${OPTARG#=}" # if long option argument, remove assigning `=`
|
|
|
|
fi
|
|
|
|
case $opt in
|
2023-04-01 20:22:30 +00:00
|
|
|
b | branch) BRANCH="$OPTARG" ;;
|
2023-03-29 01:07:18 +00:00
|
|
|
d | dir) DIR="$OPTARG" ;;
|
2023-03-29 07:47:03 +00:00
|
|
|
g | git-repo) GIT_REPO="$OPTARG" ;;
|
2023-03-29 20:26:16 +00:00
|
|
|
i | interactive) INTERACTIVE=true ;;
|
2023-03-31 23:39:28 +00:00
|
|
|
n | no-git-update) SKIP_GIT_UPDATE=true ;;
|
2023-03-29 21:41:17 +00:00
|
|
|
p | public) PUBLIC=true ;;
|
2023-03-29 07:47:03 +00:00
|
|
|
r | runpod) RUNPOD=true ;;
|
2023-03-31 01:20:42 +00:00
|
|
|
s | skip-space-check) SKIP_SPACE_CHECK=true ;;
|
|
|
|
v) ((VERBOSITY = VERBOSITY + 1)) ;;
|
2023-03-29 01:07:18 +00:00
|
|
|
h) display_help && exit 0 ;;
|
|
|
|
*) display_help && exit 0 ;;
|
|
|
|
esac
|
|
|
|
done
|
|
|
|
shift $((OPTIND - 1))
|
|
|
|
|
2023-03-31 23:39:28 +00:00
|
|
|
# Just in case someone puts in a relative path into $DIR,
|
|
|
|
# we're going to get the absolute path of that.
|
|
|
|
if [[ "$DIR" != /* ]] && [[ "$DIR" != ~* ]]; then
|
|
|
|
DIR="$(
|
|
|
|
cd "$(dirname "$DIR")" || exit 1
|
|
|
|
pwd
|
|
|
|
)/$(basename "$DIR")"
|
|
|
|
fi
|
|
|
|
|
2023-03-31 01:20:42 +00:00
|
|
|
for v in $( #Start counting from 3 since 1 and 2 are standards (stdout/stderr).
|
|
|
|
seq 3 $VERBOSITY
|
|
|
|
); do
|
|
|
|
(("$v" <= "$MAXVERBOSITY")) && eval exec "$v>&2" #Don't change anything higher than the maximum verbosity allowed.
|
|
|
|
done
|
|
|
|
|
|
|
|
for v in $( #From the verbosity level one higher than requested, through the maximum;
|
|
|
|
seq $((VERBOSITY + 1)) $MAXVERBOSITY
|
|
|
|
); do
|
|
|
|
(("$v" > "2")) && eval exec "$v>/dev/null" #Redirect these to bitbucket, provided that they don't match stdout and stderr.
|
|
|
|
done
|
|
|
|
|
|
|
|
# Example of how to use the verbosity levels.
|
|
|
|
# printf "%s\n" "This message is seen at verbosity level 1 and above." >&3
|
|
|
|
# printf "%s\n" "This message is seen at verbosity level 2 and above." >&4
|
|
|
|
# printf "%s\n" "This message is seen at verbosity level 3 and above." >&5
|
|
|
|
|
|
|
|
# Debug variable dump at max verbosity
|
|
|
|
echo "BRANCH: $BRANCH
|
|
|
|
DIR: $DIR
|
|
|
|
GIT_REPO: $GIT_REPO
|
|
|
|
INTERACTIVE: $INTERACTIVE
|
|
|
|
PUBLIC: $PUBLIC
|
|
|
|
RUNPOD: $RUNPOD
|
|
|
|
SKIP_SPACE_CHECK: $SKIP_SPACE_CHECK
|
2023-03-31 23:39:28 +00:00
|
|
|
VERBOSITY: $VERBOSITY
|
|
|
|
Script directory is ${SCRIPT_DIR}." >&5
|
2023-03-31 01:20:42 +00:00
|
|
|
|
2023-03-29 01:07:18 +00:00
|
|
|
# This must be set after the getopts loop to account for $DIR changes.
|
2023-03-29 17:53:15 +00:00
|
|
|
PARENT_DIR="$(dirname "${DIR}")"
|
2023-03-29 01:07:18 +00:00
|
|
|
VENV_DIR="$DIR/venv"
|
|
|
|
|
2023-04-01 20:28:05 +00:00
|
|
|
if [ -w "$PARENT_DIR" ] && [ ! -d "$DIR" ]; then
|
|
|
|
echo "Creating install folder ${DIR}."
|
|
|
|
mkdir "$DIR"
|
|
|
|
fi
|
|
|
|
|
|
|
|
if [ ! -w "$DIR" ]; then
|
|
|
|
echo "We cannot write to ${DIR}."
|
|
|
|
echo "Please ensure the install directory is accurate and you have the correct permissions."
|
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
|
2023-03-31 01:20:42 +00:00
|
|
|
# Shared functions
|
|
|
|
# This checks for free space on the installation drive and returns that in Gb.
|
|
|
|
size_available() {
|
|
|
|
local folder
|
|
|
|
if [ -d "$DIR" ]; then
|
|
|
|
folder="$DIR"
|
|
|
|
elif [ -d "$PARENT_DIR" ]; then
|
|
|
|
folder="$PARENT_DIR"
|
|
|
|
elif [ -d "$(echo "$DIR" | cut -d "/" -f2)" ]; then
|
|
|
|
folder="$(echo "$DIR" | cut -d "/" -f2)"
|
|
|
|
else
|
|
|
|
echo "We are assuming a root drive install for space-checking purposes."
|
|
|
|
folder='/'
|
|
|
|
fi
|
|
|
|
|
2023-03-31 23:39:28 +00:00
|
|
|
local FREESPACEINKB
|
|
|
|
FREESPACEINKB="$(df -Pk "$folder" | sed 1d | grep -v used | awk '{ print $4 "\t" }')"
|
2023-03-31 01:20:42 +00:00
|
|
|
echo "Detected available space in Kb: $FREESPACEINKB" >&5
|
2023-03-31 23:39:28 +00:00
|
|
|
local FREESPACEINGB
|
|
|
|
FREESPACEINGB=$((FREESPACEINKB / 1024 / 1024))
|
2023-03-31 01:20:42 +00:00
|
|
|
echo "$FREESPACEINGB"
|
|
|
|
}
|
|
|
|
|
2023-03-31 23:39:28 +00:00
|
|
|
# The expected usage is create_symlinks symlink target_file
|
2023-03-31 01:20:42 +00:00
|
|
|
create_symlinks() {
|
|
|
|
echo "Checking symlinks now."
|
|
|
|
# Next line checks for valid symlink
|
|
|
|
if [ -L "$1" ]; then
|
|
|
|
# Check if the linked file exists and points to the expected file
|
|
|
|
if [ -e "$1" ] && [ "$(readlink "$1")" == "$2" ]; then
|
|
|
|
echo "$(basename "$1") symlink looks fine. Skipping."
|
|
|
|
else
|
|
|
|
if [ -f "$2" ]; then
|
|
|
|
echo "Broken symlink detected. Recreating $(basename "$1")."
|
|
|
|
rm "$1" &&
|
|
|
|
ln -s "$2" "$1"
|
|
|
|
else
|
|
|
|
echo "$2 does not exist. Nothing to link."
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
else
|
|
|
|
echo "Linking $(basename "$1")."
|
|
|
|
ln -s "$2" "$1"
|
|
|
|
fi
|
|
|
|
}
|
|
|
|
|
2023-04-01 00:24:09 +00:00
|
|
|
install_python_dependencies() {
|
|
|
|
# Switch to local virtual env
|
|
|
|
echo "Switching to virtual Python environment."
|
|
|
|
if command -v python3 >/dev/null; then
|
2023-04-01 03:00:56 +00:00
|
|
|
python3 -m venv "$DIR/venv"
|
2023-04-01 00:24:09 +00:00
|
|
|
elif command -v python3.10 >/dev/null; then
|
2023-04-01 03:00:56 +00:00
|
|
|
python3.10 -m venv "$DIR/venv"
|
2023-04-01 00:24:09 +00:00
|
|
|
else
|
|
|
|
echo "Valid python3 or python3.10 binary not found."
|
|
|
|
echo "Cannot proceed with the python steps."
|
|
|
|
return 1
|
|
|
|
fi
|
|
|
|
|
|
|
|
# Activate the virtual environment
|
2023-04-01 03:00:56 +00:00
|
|
|
source "$DIR/venv/bin/activate"
|
2023-04-01 00:24:09 +00:00
|
|
|
|
2023-03-31 23:39:28 +00:00
|
|
|
# Updating pip if there is one
|
|
|
|
echo "Checking for pip updates before Python operations."
|
2023-04-01 20:28:05 +00:00
|
|
|
pip install --upgrade pip >&3
|
2023-03-31 23:39:28 +00:00
|
|
|
|
|
|
|
echo "Installing python dependencies. This could take a few minutes as it downloads files."
|
|
|
|
echo "If this operation ever runs too long, you can rerun this script in verbose mode to check."
|
|
|
|
case "$OSTYPE" in
|
|
|
|
"linux-gnu"*) pip install torch==1.12.1+cu116 torchvision==0.13.1+cu116 \
|
|
|
|
--extra-index-url https://download.pytorch.org/whl/cu116 >&3 &&
|
|
|
|
pip install -U -I --no-deps \
|
2023-04-02 01:51:24 +00:00
|
|
|
https://github.com/C43H66N12O12S2/stable-diffusion-webui/releases/download/linux/xformers-0.0.14.dev0-cp310-cp310-linux_x86_64.whl >&3 ;;
|
2023-03-31 23:39:28 +00:00
|
|
|
"darwin"*) pip install torch==2.0.0 torchvision==0.15.1 \
|
|
|
|
-f https://download.pytorch.org/whl/cpu/torch_stable.html >&3 ;;
|
|
|
|
"cygwin")
|
|
|
|
:
|
|
|
|
;;
|
|
|
|
"msys")
|
|
|
|
:
|
|
|
|
;;
|
|
|
|
esac
|
|
|
|
|
2023-04-01 00:24:09 +00:00
|
|
|
if [ "$RUNPOD" = true ]; then
|
|
|
|
echo "Installing tenssort."
|
|
|
|
pip install tensorrt >&3
|
|
|
|
fi
|
|
|
|
|
2023-03-31 23:39:28 +00:00
|
|
|
# DEBUG ONLY (Update this version number to whatever PyCharm recommends)
|
|
|
|
# pip install pydevd-pycharm~=223.8836.43
|
2023-04-01 03:00:56 +00:00
|
|
|
|
2023-04-02 19:32:56 +00:00
|
|
|
#This will copy our requirements.txt file out and make the khoya_ss lib a dynamic location then cleanup.
|
|
|
|
local TEMP_REQUIREMENTS_FILE="$DIR/requirements_tmp_for_setup.txt"
|
|
|
|
echo "Copying $DIR/requirements.txt to $TEMP_REQUIREMENTS_FILE" >&3
|
|
|
|
echo "Replacing the . for lib to our DIR variable in $TEMP_REQUIREMENTS_FILE." >&3
|
|
|
|
awk -v dir="$DIR" '/#.*kohya_ss.*library/{print; getline; sub(/^\.$/, dir)}1' "$DIR/requirements.txt" >"$TEMP_REQUIREMENTS_FILE"
|
|
|
|
|
|
|
|
# This will check if macOS is running then determine if M1+ or Intel CPU.
|
|
|
|
# It will append the appropriate packages to the requirements.txt file.
|
|
|
|
# Other OSs won't be affected and the version variables are at the top of this file.
|
|
|
|
if [[ "$(uname)" == "Darwin" ]]; then
|
|
|
|
# Check if the processor is Apple Silicon (arm64)
|
|
|
|
if [[ "$(uname -m)" == "arm64" ]]; then
|
|
|
|
echo "tensorflow-macos==$TENSORFLOW_MACOS_VERSION" >>"$TEMP_REQUIREMENTS_FILE"
|
|
|
|
echo "tensorflow-metal==$TENSORFLOW_METAL_VERSION" >>"$TEMP_REQUIREMENTS_FILE"
|
|
|
|
# Check if the processor is Intel (x86_64)
|
|
|
|
elif [[ "$(uname -m)" == "x86_64" ]]; then
|
|
|
|
echo "tensorflow==$TENSORFLOW_VERSION" >>"$TEMP_REQUIREMENTS_FILE"
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
2023-04-01 04:21:50 +00:00
|
|
|
if [ $VERBOSITY == 2 ]; then
|
2023-04-02 19:32:56 +00:00
|
|
|
python -m pip install --quiet --use-pep517 --upgrade -r "$TEMP_REQUIREMENTS_FILE" >&3
|
2023-04-01 04:21:50 +00:00
|
|
|
else
|
2023-04-02 19:32:56 +00:00
|
|
|
python -m pip install --use-pep517 --upgrade -r "$TEMP_REQUIREMENTS_FILE" >&3
|
2023-04-01 04:21:50 +00:00
|
|
|
fi
|
2023-04-01 03:00:56 +00:00
|
|
|
|
|
|
|
echo "Removing the temp requirements file."
|
2023-04-02 19:32:56 +00:00
|
|
|
if [ -f "$TEMP_REQUIREMENTS_FILE" ]; then
|
|
|
|
rm -f "$TEMP_REQUIREMENTS_FILE"
|
2023-04-01 03:00:56 +00:00
|
|
|
fi
|
2023-04-01 00:24:09 +00:00
|
|
|
|
|
|
|
if [ -n "$VIRTUAL_ENV" ]; then
|
|
|
|
if command -v deactivate >/dev/null; then
|
|
|
|
echo "Exiting Python virtual environment."
|
|
|
|
deactivate
|
|
|
|
else
|
|
|
|
echo "deactivate command not found. Could still be in the Python virtual environment."
|
|
|
|
fi
|
|
|
|
fi
|
2023-03-31 23:39:28 +00:00
|
|
|
}
|
|
|
|
|
2023-03-31 01:20:42 +00:00
|
|
|
# Attempt to non-interactively install a default accelerate config file unless specified otherwise.
|
|
|
|
# Documentation for order of precedence locations for configuration file for automated installation:
|
|
|
|
# https://huggingface.co/docs/accelerate/basic_tutorials/launch#custom-configurations
|
|
|
|
configure_accelerate() {
|
|
|
|
echo "Source accelerate config location: $DIR/config_files/accelerate/default_config.yaml" >&3
|
|
|
|
if [ "$INTERACTIVE" = true ]; then
|
|
|
|
accelerate config
|
|
|
|
else
|
|
|
|
if env_var_exists HF_HOME; then
|
|
|
|
if [ ! -f "$HF_HOME/accelerate/default_config.yaml" ]; then
|
|
|
|
mkdir -p "$HF_HOME/accelerate/" &&
|
|
|
|
echo "Target accelerate config location: $HF_HOME/accelerate/default_config.yaml" >&3
|
|
|
|
cp "$DIR/config_files/accelerate/default_config.yaml" "$HF_HOME/accelerate/default_config.yaml" &&
|
|
|
|
echo "Copied accelerate config file to: $HF_HOME/accelerate/default_config.yaml"
|
|
|
|
fi
|
|
|
|
elif env_var_exists XDG_CACHE_HOME; then
|
|
|
|
if [ ! -f "$XDG_CACHE_HOME/huggingface/accelerate" ]; then
|
|
|
|
mkdir -p "$XDG_CACHE_HOME/huggingface/accelerate" &&
|
|
|
|
echo "Target accelerate config location: $XDG_CACHE_HOME/accelerate/default_config.yaml" >&3
|
|
|
|
cp "$DIR/config_files/accelerate/default_config.yaml" "$XDG_CACHE_HOME/huggingface/accelerate/default_config.yaml" &&
|
|
|
|
echo "Copied accelerate config file to: $XDG_CACHE_HOME/huggingface/accelerate/default_config.yaml"
|
|
|
|
fi
|
|
|
|
elif env_var_exists HOME; then
|
|
|
|
if [ ! -f "$HOME/.cache/huggingface/accelerate" ]; then
|
|
|
|
mkdir -p "$HOME/.cache/huggingface/accelerate" &&
|
|
|
|
echo "Target accelerate config location: $HOME/accelerate/default_config.yaml" >&3
|
|
|
|
cp "$DIR/config_files/accelerate/default_config.yaml" "$HOME/.cache/huggingface/accelerate/default_config.yaml" &&
|
|
|
|
echo "Copying accelerate config file to: $HOME/.cache/huggingface/accelerate/default_config.yaml"
|
|
|
|
fi
|
|
|
|
else
|
|
|
|
echo "Could not place the accelerate configuration file. Please configure manually."
|
|
|
|
sleep 2
|
|
|
|
accelerate config
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
}
|
|
|
|
|
|
|
|
# Offer a warning and opportunity to cancel the installation if < 10Gb of Free Space detected
|
|
|
|
check_storage_space() {
|
|
|
|
if [ "$SKIP_SPACE_CHECK" = false ]; then
|
|
|
|
if [ "$(size_available)" -lt 10 ]; then
|
|
|
|
echo "You have less than 10Gb of free space. This installation may fail."
|
|
|
|
MSGTIMEOUT=10 # In seconds
|
|
|
|
MESSAGE="Continuing in..."
|
|
|
|
echo "Press control-c to cancel the installation."
|
2023-03-31 23:39:28 +00:00
|
|
|
for ((i = MSGTIMEOUT; i >= 0; i--)); do
|
2023-03-31 01:20:42 +00:00
|
|
|
printf "\r${MESSAGE} %ss. " "${i}"
|
|
|
|
sleep 1
|
|
|
|
done
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
}
|
|
|
|
|
2023-04-01 03:00:56 +00:00
|
|
|
# These are the git operations that will run to update or clone the repo
|
2023-03-31 23:39:28 +00:00
|
|
|
update_kohya_ss() {
|
|
|
|
if [ "$SKIP_GIT_UPDATE" = false ]; then
|
|
|
|
if command -v git >/dev/null; then
|
|
|
|
# First, we make sure there are no changes that need to be made in git, so no work is lost.
|
2023-04-01 04:11:24 +00:00
|
|
|
if [ "$(git -C "$DIR" status --porcelain=v1 2>/dev/null | wc -l)" -gt 0 ] &&
|
2023-04-01 04:03:25 +00:00
|
|
|
echo "These files need to be committed or discarded: " >&4 &&
|
|
|
|
git -C "$DIR" status >&4; then
|
2023-04-01 03:51:49 +00:00
|
|
|
echo "There are changes that need to be committed or discarded in the repo in $DIR."
|
2023-03-31 23:39:28 +00:00
|
|
|
echo "Commit those changes or run this script with -n to skip git operations entirely."
|
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
|
2023-04-01 20:22:30 +00:00
|
|
|
echo "Attempting to clone $GIT_REPO."
|
2023-03-31 23:39:28 +00:00
|
|
|
if [ ! -d "$DIR/.git" ]; then
|
2023-04-01 20:28:05 +00:00
|
|
|
echo "Cloning and switching to $GIT_REPO:$BRANCH" >&4
|
|
|
|
git -C "$PARENT_DIR" clone -b "$BRANCH" "$GIT_REPO" "$(basename "$DIR")" >&3
|
2023-04-01 03:54:21 +00:00
|
|
|
git -C "$DIR" switch "$BRANCH" >&4
|
2023-03-31 23:39:28 +00:00
|
|
|
else
|
|
|
|
echo "git repo detected. Attempting to update repository instead."
|
|
|
|
echo "Updating: $GIT_REPO"
|
2023-04-01 20:22:30 +00:00
|
|
|
git -C "$DIR" pull "$GIT_REPO" "$BRANCH" >&3
|
|
|
|
if ! git -C "$DIR" switch "$BRANCH" >&4; then
|
|
|
|
echo "Branch $BRANCH did not exist. Creating it." >&4
|
|
|
|
git -C "$DIR" switch -c "$BRANCH" >&4
|
2023-04-01 17:15:15 +00:00
|
|
|
fi
|
2023-03-31 23:39:28 +00:00
|
|
|
fi
|
|
|
|
else
|
|
|
|
echo "You need to install git."
|
|
|
|
echo "Rerun this after installing git or run this script with -n to skip the git operations."
|
|
|
|
fi
|
2023-04-01 00:24:09 +00:00
|
|
|
else
|
|
|
|
echo "Skipping git operations."
|
2023-03-31 23:39:28 +00:00
|
|
|
fi
|
|
|
|
}
|
|
|
|
|
2023-03-31 01:20:42 +00:00
|
|
|
# Start OS-specific detection and work
|
2023-03-27 19:37:00 +00:00
|
|
|
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
|
|
|
|
# Check if root or sudo
|
2023-03-29 19:07:00 +00:00
|
|
|
root=false
|
|
|
|
if [ "$EUID" = 0 ]; then
|
|
|
|
root=true
|
|
|
|
elif command -v id >/dev/null && [ "$(id -u)" = 0 ]; then
|
|
|
|
root=true
|
|
|
|
elif [ "$UID" = 0 ]; then
|
|
|
|
root=true
|
2023-03-27 19:37:00 +00:00
|
|
|
fi
|
|
|
|
|
2023-03-28 22:52:16 +00:00
|
|
|
get_distro_name() {
|
|
|
|
local line
|
|
|
|
if [ -f /etc/os-release ]; then
|
|
|
|
# We search for the line starting with ID=
|
|
|
|
# Then we remove the ID= prefix to get the name itself
|
|
|
|
line="$(grep -Ei '^ID=' /etc/os-release)"
|
2023-03-31 01:20:42 +00:00
|
|
|
echo "Raw detected os-release distro line: $line" >&5
|
2023-03-28 22:52:16 +00:00
|
|
|
line=${line##*=}
|
|
|
|
echo "$line"
|
|
|
|
return 0
|
|
|
|
elif command -v python >/dev/null; then
|
|
|
|
line="$(python -mplatform)"
|
|
|
|
echo "$line"
|
|
|
|
return 0
|
|
|
|
elif command -v python3 >/dev/null; then
|
|
|
|
line="$(python3 -mplatform)"
|
|
|
|
echo "$line"
|
|
|
|
return 0
|
|
|
|
else
|
|
|
|
line="None"
|
|
|
|
echo "$line"
|
|
|
|
return 1
|
|
|
|
fi
|
|
|
|
}
|
|
|
|
|
2023-04-01 03:00:56 +00:00
|
|
|
# We search for the line starting with ID_LIKE=
|
|
|
|
# Then we remove the ID_LIKE= prefix to get the name itself
|
|
|
|
# This is the "type" of distro. For example, Ubuntu returns "debian".
|
2023-03-28 22:52:16 +00:00
|
|
|
get_distro_family() {
|
|
|
|
local line
|
|
|
|
if [ -f /etc/os-release ]; then
|
|
|
|
if grep -Eiq '^ID_LIKE=' /etc/os-release >/dev/null; then
|
|
|
|
line="$(grep -Ei '^ID_LIKE=' /etc/os-release)"
|
2023-03-31 01:20:42 +00:00
|
|
|
echo "Raw detected os-release distro family line: $line" >&5
|
2023-03-28 22:52:16 +00:00
|
|
|
line=${line##*=}
|
|
|
|
echo "$line"
|
|
|
|
return 0
|
|
|
|
else
|
|
|
|
line="None"
|
|
|
|
echo "$line"
|
|
|
|
return 1
|
|
|
|
fi
|
|
|
|
else
|
|
|
|
line="None"
|
|
|
|
echo "$line"
|
|
|
|
return 1
|
|
|
|
fi
|
|
|
|
}
|
|
|
|
|
2023-03-31 01:20:42 +00:00
|
|
|
check_storage_space
|
2023-03-31 23:39:28 +00:00
|
|
|
update_kohya_ss
|
|
|
|
|
2023-03-28 22:52:16 +00:00
|
|
|
distro=get_distro_name
|
|
|
|
family=get_distro_family
|
2023-03-31 01:20:42 +00:00
|
|
|
echo "Raw detected distro string: $distro" >&4
|
|
|
|
echo "Raw detected distro family string: $family" >&4
|
2023-03-29 18:42:36 +00:00
|
|
|
|
|
|
|
echo "Installing Python TK if not found on the system."
|
2023-03-28 22:52:16 +00:00
|
|
|
if "$distro" | grep -qi "Ubuntu" || "$family" | grep -qi "Ubuntu"; then
|
2023-03-27 19:37:00 +00:00
|
|
|
echo "Ubuntu detected."
|
2023-03-29 18:42:36 +00:00
|
|
|
if [ $(dpkg-query -W -f='${Status}' python3-tk 2>/dev/null | grep -c "ok installed") = 0 ]; then
|
2023-03-29 19:07:00 +00:00
|
|
|
if [ "$root" = true ]; then
|
2023-03-31 01:20:42 +00:00
|
|
|
apt update -y >&3 && apt install -y python3-tk >&3
|
2023-03-27 19:37:00 +00:00
|
|
|
else
|
|
|
|
echo "This script needs to be run as root or via sudo to install packages."
|
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
else
|
|
|
|
echo "Python TK found! Skipping install!"
|
|
|
|
fi
|
|
|
|
elif "$distro" | grep -Eqi "Fedora|CentOS|Redhat"; then
|
2023-03-29 18:42:36 +00:00
|
|
|
echo "Redhat or Redhat base detected."
|
2023-03-27 19:37:00 +00:00
|
|
|
if ! rpm -qa | grep -qi python3-tkinter; then
|
2023-03-29 19:07:00 +00:00
|
|
|
if [ "$root" = true ]; then
|
2023-03-31 01:20:42 +00:00
|
|
|
dnf install python3-tkinter -y >&3
|
2023-03-27 19:37:00 +00:00
|
|
|
else
|
|
|
|
echo "This script needs to be run as root or via sudo to install packages."
|
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
fi
|
2023-03-28 22:52:16 +00:00
|
|
|
elif "$distro" | grep -Eqi "arch" || "$family" | grep -qi "arch"; then
|
2023-03-29 18:42:36 +00:00
|
|
|
echo "Arch Linux or Arch base detected."
|
2023-03-28 22:52:16 +00:00
|
|
|
if ! pacman -Qi tk >/dev/null; then
|
2023-03-29 19:07:00 +00:00
|
|
|
if [ "$root" = true ]; then
|
2023-03-31 01:20:42 +00:00
|
|
|
pacman --noconfirm -S tk >&3
|
2023-03-28 22:52:16 +00:00
|
|
|
else
|
|
|
|
echo "This script needs to be run as root or via sudo to install packages."
|
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
elif "$distro" | grep -Eqi "opensuse" || "$family" | grep -qi "opensuse"; then
|
2023-03-29 18:42:36 +00:00
|
|
|
echo "OpenSUSE detected."
|
2023-03-28 22:52:16 +00:00
|
|
|
if ! rpm -qa | grep -qi python-tk; then
|
2023-03-29 19:07:00 +00:00
|
|
|
if [ "$root" = true ]; then
|
2023-03-31 01:20:42 +00:00
|
|
|
zypper install -y python-tk >&3
|
2023-03-28 22:52:16 +00:00
|
|
|
else
|
|
|
|
echo "This script needs to be run as root or via sudo to install packages."
|
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
elif [ "$distro" = "None" ] || [ "$family" = "None" ]; then
|
|
|
|
if [ "$distro" = "None" ]; then
|
|
|
|
echo "We could not detect your distribution of Linux. Please file a bug report on github with the contents of your /etc/os-release file."
|
|
|
|
fi
|
|
|
|
|
|
|
|
if [ "$family" = "None" ]; then
|
|
|
|
echo "We could not detect the family of your Linux distribution. Please file a bug report on github with the contents of your /etc/os-release file."
|
|
|
|
fi
|
2023-03-27 19:37:00 +00:00
|
|
|
fi
|
|
|
|
|
2023-04-01 00:24:09 +00:00
|
|
|
install_python_dependencies
|
2023-03-29 01:07:18 +00:00
|
|
|
|
2023-04-01 00:24:09 +00:00
|
|
|
# We need just a little bit more setup for non-interactive environments
|
2023-03-29 07:47:03 +00:00
|
|
|
if [ "$RUNPOD" = true ]; then
|
2023-03-31 01:20:42 +00:00
|
|
|
# Symlink paths
|
|
|
|
libnvinfer_plugin_symlink="$VENV_DIR/lib/python3.10/site-packages/tensorrt/libnvinfer_plugin.so.7"
|
|
|
|
libnvinfer_symlink="$VENV_DIR/lib/python3.10/site-packages/tensorrt/libnvinfer.so.7"
|
|
|
|
libcudart_symlink="$VENV_DIR/lib/python3.10/site-packages/nvidia/cuda_runtime/lib/libcudart.so.11.0"
|
|
|
|
|
|
|
|
#Target file paths
|
|
|
|
libnvinfer_plugin_target="$VENV_DIR/lib/python3.10/site-packages/tensorrt/libnvinfer_plugin.so.8"
|
|
|
|
libnvinfer_target="$VENV_DIR/lib/python3.10/site-packages/tensorrt/libnvinfer.so.8"
|
|
|
|
libcudart_target="$VENV_DIR/lib/python3.10/site-packages/nvidia/cuda_runtime/lib/libcudart.so.12"
|
|
|
|
|
|
|
|
echo "Checking symlinks now."
|
|
|
|
create_symlinks "$libnvinfer_plugin_symlink" "$libnvinfer_plugin_target"
|
|
|
|
create_symlinks "$libnvinfer_symlink" "$libnvinfer_target"
|
|
|
|
create_symlinks "$libcudart_symlink" "$libcudart_target"
|
2023-03-29 01:07:18 +00:00
|
|
|
|
2023-04-01 20:28:05 +00:00
|
|
|
if [ -d "${VENV_DIR}/lib/python3.10/site-packages/tensorrt/" ]; then
|
|
|
|
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${VENV_DIR}/lib/python3.10/site-packages/tensorrt/"
|
|
|
|
else
|
|
|
|
echo "${VENV_DIR}/lib/python3.10/site-packages/tensorrt/ not found; not linking library."
|
|
|
|
fi
|
|
|
|
|
|
|
|
if [ -d "${VENV_DIR}/lib/python3.10/site-packages/tensorrt/" ]; then
|
|
|
|
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${VENV_DIR}/lib/python3.10/site-packages/nvidia/cuda_runtime/lib/"
|
|
|
|
else
|
|
|
|
echo "${VENV_DIR}/lib/python3.10/site-packages/nvidia/cuda_runtime/lib/ not found; not linking library."
|
|
|
|
fi
|
2023-03-29 01:07:18 +00:00
|
|
|
|
2023-03-31 01:20:42 +00:00
|
|
|
configure_accelerate
|
2023-03-29 20:26:16 +00:00
|
|
|
|
2023-03-29 01:07:18 +00:00
|
|
|
# This is a non-interactive environment, so just directly call gui.sh after all setup steps are complete.
|
|
|
|
if command -v bash >/dev/null; then
|
2023-03-29 21:41:17 +00:00
|
|
|
if [ "$PUBLIC" = false ]; then
|
|
|
|
bash "$DIR"/gui.sh
|
|
|
|
else
|
|
|
|
bash "$DIR"/gui.sh --share
|
|
|
|
fi
|
2023-03-29 01:07:18 +00:00
|
|
|
else
|
|
|
|
# This shouldn't happen, but we're going to try to help.
|
2023-03-29 21:41:17 +00:00
|
|
|
if [ "$PUBLIC" = false ]; then
|
|
|
|
sh "$DIR"/gui.sh
|
|
|
|
else
|
|
|
|
sh "$DIR"/gui.sh --share
|
|
|
|
fi
|
2023-03-29 01:07:18 +00:00
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
2023-03-27 19:37:00 +00:00
|
|
|
echo -e "Setup finished! Run \e[0;92m./gui.sh\e[0m to start."
|
2023-03-31 01:20:42 +00:00
|
|
|
echo "Please note if you'd like to expose your public server you need to run ./gui.sh --share"
|
2023-03-27 19:37:00 +00:00
|
|
|
elif [[ "$OSTYPE" == "darwin"* ]]; then
|
|
|
|
# The initial setup script to prep the environment on macOS
|
|
|
|
# xformers has been omitted as that is for Nvidia GPUs only
|
|
|
|
|
|
|
|
if ! command -v brew >/dev/null; then
|
|
|
|
echo "Please install homebrew first. This is a requirement for the remaining setup."
|
|
|
|
echo "You can find that here: https://brew.sh"
|
2023-03-28 22:52:16 +00:00
|
|
|
#shellcheck disable=SC2016
|
|
|
|
echo 'The "brew" command should be in $PATH to be detected.'
|
2023-03-27 19:37:00 +00:00
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
|
2023-03-31 01:20:42 +00:00
|
|
|
check_storage_space
|
|
|
|
|
2023-03-27 19:37:00 +00:00
|
|
|
# Install base python packages
|
|
|
|
echo "Installing Python 3.10 if not found."
|
|
|
|
if ! brew ls --versions python@3.10 >/dev/null; then
|
2023-03-31 01:20:42 +00:00
|
|
|
echo "Installing Python 3.10."
|
|
|
|
brew install python@3.10 >&3
|
2023-03-27 19:37:00 +00:00
|
|
|
else
|
|
|
|
echo "Python 3.10 found!"
|
|
|
|
fi
|
|
|
|
echo "Installing Python-TK 3.10 if not found."
|
|
|
|
if ! brew ls --versions python-tk@3.10 >/dev/null; then
|
2023-03-31 01:20:42 +00:00
|
|
|
echo "Installing Python TK 3.10."
|
|
|
|
brew install python-tk@3.10 >&3
|
2023-03-27 19:37:00 +00:00
|
|
|
else
|
|
|
|
echo "Python Tkinter 3.10 found!"
|
|
|
|
fi
|
|
|
|
|
2023-03-31 23:39:28 +00:00
|
|
|
update_kohya_ss
|
|
|
|
|
2023-04-01 00:24:09 +00:00
|
|
|
if ! install_python_dependencies; then
|
|
|
|
echo "You may need to install Python. The command for this is brew install python@3.10."
|
2023-03-27 19:37:00 +00:00
|
|
|
fi
|
2023-04-01 00:24:09 +00:00
|
|
|
|
|
|
|
configure_accelerate
|
|
|
|
echo -e "Setup finished! Run ./gui.sh to start."
|
2023-03-27 19:37:00 +00:00
|
|
|
elif [[ "$OSTYPE" == "cygwin" ]]; then
|
2023-04-02 19:35:47 +00:00
|
|
|
# Cygwin is a standalone suite of Linux utilities on Windows
|
2023-03-27 19:37:00 +00:00
|
|
|
echo "This hasn't been validated on cygwin yet."
|
|
|
|
elif [[ "$OSTYPE" == "msys" ]]; then
|
2023-04-02 19:35:47 +00:00
|
|
|
# MinGW has the msys environment which is a standalone suite of Linux utilities on Windows
|
2023-03-27 19:37:00 +00:00
|
|
|
# "git bash" on Windows may also be detected as msys.
|
|
|
|
echo "This hasn't been validated in msys (mingw) on Windows yet."
|
2023-04-02 19:32:56 +00:00
|
|
|
fi
|