From 97b004e756e3cec6195c87d313b98276f1a98a83 Mon Sep 17 00:00:00 2001 From: bmaltais Date: Sat, 1 Apr 2023 06:29:45 -0400 Subject: [PATCH] Revert "Merge pull request #466 from jstayco/consolidated_install_scripts" This reverts commit b7a719b51a56bb4094512c76f48ef118649874c6, reversing changes made to 538752ccab9032d9fd7035bfb5ba2583b30747ca. --- .gitignore | 3 +- README.md | 155 ++---- config_files/accelerate/default_config.yaml | 22 - gui.sh | 12 +- gui_macos.sh | 13 + macos_setup.sh | 38 ++ requirements.txt | 9 +- requirements_macos.txt | 32 ++ setup.sh | 578 -------------------- ubuntu_setup.sh | 12 + upgrade.sh | 16 + upgrade_macos.sh | 16 + 12 files changed, 184 insertions(+), 722 deletions(-) delete mode 100644 config_files/accelerate/default_config.yaml create mode 100755 gui_macos.sh create mode 100755 macos_setup.sh create mode 100644 requirements_macos.txt delete mode 100755 setup.sh create mode 100755 ubuntu_setup.sh create mode 100755 upgrade.sh create mode 100755 upgrade_macos.sh diff --git a/.gitignore b/.gitignore index 5ed6370..71fe116 100644 --- a/.gitignore +++ b/.gitignore @@ -8,5 +8,4 @@ wd14_tagger_model .DS_Store locon gui-user.bat -gui-user.ps1 -.idea \ No newline at end of file +gui-user.ps1 \ No newline at end of file diff --git a/README.md b/README.md index 04018cc..3614d94 100644 --- a/README.md +++ b/README.md @@ -6,30 +6,21 @@ If you run on Linux and would like to use the GUI, there is now a port of it as ### Table of Contents -- [Tutorials](#tutorials) -- [Required Dependencies](#required-dependencies) - - [Linux/macOS](#linux-and-macos-dependencies) -- [Installation](#installation) - - [Linux/macOS](#linux-and-macos) - - [Default Install Locations](#install-location) - - [Windows](#windows) - - [CUDNN 8.6](#optional--cudnn-86) -- [Upgrading](#upgrading) - - [Windows](#windows-upgrade) - - [Linux/macOS](#linux-and-macos-upgrade) -- [Launching the GUI](#starting-gui-service) - - [Windows](#launching-the-gui-on-windows) - - [Linux/macOS](#launching-the-gui-on-linux-and-macos) - - [Direct Launch via Python Script](#launching-the-gui-directly-using-kohyaguipy) -- [Dreambooth](#dreambooth) -- [Finetune](#finetune) -- [Train Network](#train-network) -- [LoRA](#lora) -- [Troubleshooting](#troubleshooting) - - [Page File Limit](#page-file-limit) - - [No module called tkinter](#no-module-called-tkinter) - - [FileNotFoundError](#filenotfounderror) -- [Change History](#change-history) +- [Tutorials](https://github.com/bmaltais/kohya_ss#tutorials) +- [Required Dependencies](https://github.com/bmaltais/kohya_ss#required-dependencies) +- [Installation](https://github.com/bmaltais/kohya_ss#installation) + - [CUDNN 8.6](https://github.com/bmaltais/kohya_ss#optional-cudnn-86) +- [Upgrading](https://github.com/bmaltais/kohya_ss#upgrading) +- [Launching the GUI](https://github.com/bmaltais/kohya_ss#launching-the-gui) +- [Dreambooth](https://github.com/bmaltais/kohya_ss#dreambooth) +- [Finetune](https://github.com/bmaltais/kohya_ss#finetune) +- [Train Network](https://github.com/bmaltais/kohya_ss#train-network) +- [LoRA](https://github.com/bmaltais/kohya_ss#lora) +- [Troubleshooting](https://github.com/bmaltais/kohya_ss#troubleshooting) + - [Page File Limit](https://github.com/bmaltais/kohya_ss#page-file-limit) + - [No module called tkinter](https://github.com/bmaltais/kohya_ss#no-module-called-tkinter) + - [FileNotFoundError](https://github.com/bmaltais/kohya_ss#filenotfounderror) +- [Change History](https://github.com/bmaltais/kohya_ss#change-history) ## Tutorials @@ -48,66 +39,35 @@ If you run on Linux and would like to use the GUI, there is now a port of it as - Install [Git](https://git-scm.com/download/win) - Install [Visual Studio 2015, 2017, 2019, and 2022 redistributable](https://aka.ms/vs/17/release/vc_redist.x64.exe) -### Linux and macOS dependencies - -These dependencies are taken care of via `setup.sh` in the installation section. No additional steps should be needed unless the scripts inform you otherwise. - ## Installation ### Runpod Follow the instructions found in this discussion: https://github.com/bmaltais/kohya_ss/discussions/379 -### Linux and macOS +### MacOS In the terminal, run ``` git clone https://github.com/bmaltais/kohya_ss.git cd kohya_ss -# May need to chmod +x ./setup.sh if you're on a machine with stricter security. -# There are additional options if needed for a runpod environment. -# Call 'setup.sh -h' or 'setup.sh --help' for more information. -./setup.sh +bash macos_setup.sh ``` -Setup.sh help included here: +During the accelerate config screen after running the script answer "This machine", "None", "No" for the remaining questions. -```bash -Kohya_SS Installation Script for POSIX operating systems. +### Ubuntu +In the terminal, run -The following options are useful in a runpod environment, -but will not affect a local machine install. - -Usage: - setup.sh -b dev -d /workspace/kohya_ss -g https://mycustom.repo.tld/custom_fork.git - setup.sh --branch=dev --dir=/workspace/kohya_ss --git-repo=https://mycustom.repo.tld/custom_fork.git - -Options: - -b BRANCH, --branch=BRANCH Select which branch of kohya to check out on new installs. - -d DIR, --dir=DIR The full path you want kohya_ss installed to. - -g REPO, --git_repo=REPO You can optionally provide a git repo to check out for runpod installation. Useful for custom forks. - -h, --help Show this screen. - -i, --interactive Interactively configure accelerate instead of using default config file. - -n, --no-update Do not update kohya_ss repo. No git pull or clone operations. - -p, --public Expose public URL in runpod mode. Won't have an effect in other modes. - -r, --runpod Forces a runpod installation. Useful if detection fails for any reason. - -s, --skip-space-check Skip the 10Gb minimum storage space check. - -v, --verbose Increase verbosity levels up to 3. +``` +git clone https://github.com/bmaltais/kohya_ss.git +cd kohya_ss +bash ubuntu_setup.sh ``` -#### Install location - -The default install location for Linux is where the script is located if a previous installation is detected that location. -Otherwise, it will fall to `/opt/kohya_ss`. If /opt is not writeable, the fallback is `$HOME/kohya_ss`. Lastly, if all else fails it will simply install to the current folder you are in (PWD). - -On macOS and other non-Linux machines, it will first try to detect an install where the script is run from and then run setup there if that's detected. -If a previous install isn't found at that location, then it will default install to `$HOME/kohya_ss` followed by where you're currently at if there's no access to $HOME. -You can override this behavior by specifying an install directory with the -d option. - -If you are using the interactive mode, our default values for the accelerate config screen after running the script answer "This machine", "None", "No" for the remaining questions. -These are the same answers as the Windows install. +then configure accelerate with the same answers as in the MacOS instructions when prompted. ### Windows -In the terminal, run: +In the terminal, run ``` git clone https://github.com/bmaltais/kohya_ss.git @@ -115,7 +75,7 @@ cd kohya_ss setup.bat ``` -Then configure accelerate with the same answers as in the MacOS instructions when prompted. +then configure accelerate with the same answers as in the MacOS instructions when prompted. ### Optional: CUDNN 8.6 @@ -133,58 +93,38 @@ Run the following commands to install: python .\tools\cudann_1.8_install.py ``` -Once the commands have completed successfully you should be ready to use the new version. MacOS support is not tested and has been mostly taken from https://gist.github.com/jstayco/9f5733f05b9dc29de95c4056a023d645 +## Upgrading MacOS -## Upgrading +When a new release comes out, you can upgrade your repo with the following commands in the root directory: -The following commands will work from the root directory of the project if you'd prefer to not run scripts. -These commands will work on any OS. ```bash -git pull - -.\venv\Scripts\activate - -pip install --use-pep517 --upgrade -r requirements.txt +upgrade_macos.sh ``` -### Windows Upgrade +Once the commands have completed successfully you should be ready to use the new version. MacOS support is not tested and has been mostly taken from https://gist.github.com/jstayco/9f5733f05b9dc29de95c4056a023d645 + +## Upgrading Windows + When a new release comes out, you can upgrade your repo with the following commands in the root directory: ```powershell upgrade.bat ``` -### Linux and macOS Upgrade -You can cd into the root directory and simply run - -```bash -# Refresh and update everything -./setup.sh - -# This will refresh everything, but NOT clone or pull the git repo. -./setup.sh --no-git-update -``` - Once the commands have completed successfully you should be ready to use the new version. -# Starting GUI Service +## Launching the GUI using gui.bat or gui.ps1 + +The script can be run with several optional command line arguments: -The following command line arguments can be passed to the scripts on any OS to configure the underlying service. -``` --listen: the IP address to listen on for connections to Gradio. ---username: a username for authentication. ---password: a password for authentication. ---server_port: the port to run the server listener on. ---inbrowser: opens the Gradio UI in a web browser. +--username: a username for authentication. +--password: a password for authentication. +--server_port: the port to run the server listener on. +--inbrowser: opens the Gradio UI in a web browser. --share: shares the Gradio UI. -``` -### Launching the GUI on Windows - -The two scripts to launch the GUI on Windows are gui.ps1 and gui.bat in the root directory. -You can use whichever script you prefer. - -To launch the Gradio UI, run the script in a terminal with the desired command line arguments, for example: +These command line arguments can be passed to the UI function as keyword arguments. To launch the Gradio UI, run the script in a terminal with the desired command line arguments, for example: `gui.ps1 --listen 127.0.0.1 --server_port 7860 --inbrowser --share` @@ -192,19 +132,14 @@ or `gui.bat --listen 127.0.0.1 --server_port 7860 --inbrowser --share` -## Launching the GUI on Linux and macOS +## Launching the GUI using kohya_gui.py -Run the launcher script with the desired command line arguments similar to Windows. -`gui.sh --listen 127.0.0.1 --server_port 7860 --inbrowser --share` - -## Launching the GUI directly using kohya_gui.py - -To run the GUI directly bypassing the wrapper scripts, simply use this command from the root project directory: +To run the GUI, simply use this command: ``` .\venv\Scripts\activate -python .\kohya_gui.py +python.exe .\kohya_gui.py ``` ## Dreambooth diff --git a/config_files/accelerate/default_config.yaml b/config_files/accelerate/default_config.yaml deleted file mode 100644 index a31ddd0..0000000 --- a/config_files/accelerate/default_config.yaml +++ /dev/null @@ -1,22 +0,0 @@ -command_file: null -commands: null -compute_environment: LOCAL_MACHINE -deepspeed_config: {} -distributed_type: 'NO' -downcast_bf16: 'no' -dynamo_backend: 'NO' -fsdp_config: {} -gpu_ids: all -machine_rank: 0 -main_process_ip: null -main_process_port: null -main_training_function: main -megatron_lm_config: {} -mixed_precision: 'no' -num_machines: 1 -num_processes: 1 -rdzv_backend: static -same_network: true -tpu_name: null -tpu_zone: null -use_cpu: false diff --git a/gui.sh b/gui.sh index 4fa2e35..e4eca6f 100755 --- a/gui.sh +++ b/gui.sh @@ -1,9 +1,13 @@ -#!/usr/bin/env bash +#!/bin/bash # Activate the virtual environment -source ./venv/bin/activate +source venv/bin/activate -# If the requirements are validated, run the kohya_gui.py script with the command-line arguments -if python tools/validate_requirements.py; then +# Validate the requirements and store the exit code +python tools/validate_requirements.py +exit_code=$? + +# If the exit code is 0, run the kohya_gui.py script with the command-line arguments +if [ $exit_code -eq 0 ]; then python kohya_gui.py "$@" fi diff --git a/gui_macos.sh b/gui_macos.sh new file mode 100755 index 0000000..4a0bfb8 --- /dev/null +++ b/gui_macos.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# Activate the virtual environment +source venv/bin/activate + +# Validate the requirements and store the exit code +python tools/validate_requirements.py --requirements requirements_macos.txt +exit_code=$? + +# If the exit code is 0, run the kohya_gui.py script with the command-line arguments +if [ $exit_code -eq 0 ]; then + python kohya_gui.py "$@" +fi diff --git a/macos_setup.sh b/macos_setup.sh new file mode 100755 index 0000000..4de8417 --- /dev/null +++ b/macos_setup.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# The initial setup script to prep the environment on macOS +# xformers has been omitted as that is for Nvidia GPUs only + +if ! command -v brew >/dev/null; then + echo "Please install homebrew first. This is a requirement for the remaining setup." + echo "You can find that here: https://brew.sh" + exit 1 +fi + +# Install base python packages +echo "Installing Python 3.10 if not found." +brew ls --versions python@3.10 >/dev/null || brew install python@3.10 +echo "Installing Python-TK 3.10 if not found." +brew ls --versions python-tk@3.10 >/dev/null || brew install python-tk@3.10 + +if command -v python3.10 >/dev/null; then + python3.10 -m venv venv + source venv/bin/activate + + # DEBUG ONLY + #pip install pydevd-pycharm~=223.8836.43 + + # Tensorflow installation + if wget https://github.com/apple/tensorflow_macos/releases/download/v0.1alpha3/tensorflow_macos-0.1a3-cp38-cp38-macosx_11_0_arm64.whl /tmp; then + python -m pip install tensorflow==0.1a3 -f https://github.com/apple/tensorflow_macos/releases/download/v0.1alpha3/tensorflow_macos-0.1a3-cp38-cp38-macosx_11_0_arm64.whl + rm -f /tmp/tensorflow_macos-0.1a3-cp38-cp38-macosx_11_0_arm64.whl + fi + + pip install torch==2.0.0 torchvision==0.15.1 -f https://download.pytorch.org/whl/cpu/torch_stable.html + python -m pip install --use-pep517 --upgrade -r requirements_macos.txt + accelerate config + echo -e "Setup finished! Run ./gui_macos.sh to start." +else + echo "Python not found. Please ensure you install Python." + echo "The brew command for Python 3.10 is: brew install python@3.10" + exit 1 +fi \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 01f6503..5881d6e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,14 +7,12 @@ diffusers[torch]==0.10.2 easygui==0.98.3 einops==0.6.0 ftfy==6.1.1 -gradio==3.19.1; sys_platform != 'darwin' -gradio==3.23.0; sys_platform == 'darwin' +gradio==3.19.1 lion-pytorch==0.0.6 opencv-python==4.7.0.68 pytorch-lightning==1.9.0 safetensors==0.2.6 -tensorboard==2.10.1 ; sys_platform != 'darwin' -tensorboard==2.12.1 ; sys_platform == 'darwin' +tensorboard==2.10.1 tk==0.1.0 toml==0.10.2 transformers==4.26.0 @@ -25,8 +23,7 @@ requests==2.28.2 timm==0.6.12 # tensorflow<2.11 huggingface-hub==0.13.0 -tensorflow==2.10.1; sys_platform != 'darwin' -tensorflow-macos==2.12.0; sys_platform == 'darwin' +tensorflow==2.10.1 # For locon support lycoris-lora @ git+https://github.com/KohakuBlueleaf/LyCORIS.git@c3d925421209a22a60d863ffa3de0b3e7e89f047 # lycoris_lora==0.1.4 diff --git a/requirements_macos.txt b/requirements_macos.txt new file mode 100644 index 0000000..4ee4eec --- /dev/null +++ b/requirements_macos.txt @@ -0,0 +1,32 @@ +accelerate==0.15.0 +albumentations==1.3.0 +altair==4.2.2 +bitsandbytes==0.35.0 +dadaptation==1.5 +diffusers[torch]==0.10.2 +easygui==0.98.3 +einops==0.6.0 +ftfy==6.1.1 +gradio==3.19.1; sys_platform != 'darwin' +gradio==3.23.0; sys_platform == 'darwin' +lion-pytorch==0.0.6 +opencv-python==4.7.0.68 +pytorch-lightning==1.9.0 +safetensors==0.2.6 +tensorboard==2.10.1 +tk==0.1.0 +toml==0.10.2 +transformers==4.26.0 +voluptuous==0.13.1 +# for BLIP captioning +fairscale==0.4.13 +requests==2.28.2 +timm==0.6.12 +# tensorflow<2.11 +huggingface-hub==0.12.0; sys_platform != 'darwin' +huggingface-hub==0.13.0; sys_platform == 'darwin' +tensorflow==2.10.1; sys_platform != 'darwin' +# For locon support +lycoris_lora==0.1.2 +# for kohya_ss library +. \ No newline at end of file diff --git a/setup.sh b/setup.sh deleted file mode 100755 index bc98d6e..0000000 --- a/setup.sh +++ /dev/null @@ -1,578 +0,0 @@ -#!/usr/bin/env bash - -# This file will be the host environment setup file for all operating systems other than base Windows. - -display_help() { - cat <&2" #Don't change anything higher than the maximum verbosity allowed. -done - -for v in $( #From the verbosity level one higher than requested, through the maximum; - seq $((VERBOSITY + 1)) $MAXVERBOSITY -); do - (("$v" > "2")) && eval exec "$v>/dev/null" #Redirect these to bitbucket, provided that they don't match stdout and stderr. -done - -# Example of how to use the verbosity levels. -# printf "%s\n" "This message is seen at verbosity level 1 and above." >&3 -# printf "%s\n" "This message is seen at verbosity level 2 and above." >&4 -# printf "%s\n" "This message is seen at verbosity level 3 and above." >&5 - -# Debug variable dump at max verbosity -echo "BRANCH: $BRANCH -DIR: $DIR -GIT_REPO: $GIT_REPO -INTERACTIVE: $INTERACTIVE -PUBLIC: $PUBLIC -RUNPOD: $RUNPOD -SKIP_SPACE_CHECK: $SKIP_SPACE_CHECK -VERBOSITY: $VERBOSITY -Script directory is ${SCRIPT_DIR}." >&5 - -# This must be set after the getopts loop to account for $DIR changes. -PARENT_DIR="$(dirname "${DIR}")" -VENV_DIR="$DIR/venv" - -# Shared functions -# This checks for free space on the installation drive and returns that in Gb. -size_available() { - local folder - if [ -d "$DIR" ]; then - folder="$DIR" - elif [ -d "$PARENT_DIR" ]; then - folder="$PARENT_DIR" - elif [ -d "$(echo "$DIR" | cut -d "/" -f2)" ]; then - folder="$(echo "$DIR" | cut -d "/" -f2)" - else - echo "We are assuming a root drive install for space-checking purposes." - folder='/' - fi - - local FREESPACEINKB - FREESPACEINKB="$(df -Pk "$folder" | sed 1d | grep -v used | awk '{ print $4 "\t" }')" - echo "Detected available space in Kb: $FREESPACEINKB" >&5 - local FREESPACEINGB - FREESPACEINGB=$((FREESPACEINKB / 1024 / 1024)) - echo "$FREESPACEINGB" -} - -# The expected usage is create_symlinks symlink target_file -create_symlinks() { - echo "Checking symlinks now." - # Next line checks for valid symlink - if [ -L "$1" ]; then - # Check if the linked file exists and points to the expected file - if [ -e "$1" ] && [ "$(readlink "$1")" == "$2" ]; then - echo "$(basename "$1") symlink looks fine. Skipping." - else - if [ -f "$2" ]; then - echo "Broken symlink detected. Recreating $(basename "$1")." - rm "$1" && - ln -s "$2" "$1" - else - echo "$2 does not exist. Nothing to link." - fi - fi - else - echo "Linking $(basename "$1")." - ln -s "$2" "$1" - fi -} - -install_python_dependencies() { - # Switch to local virtual env - echo "Switching to virtual Python environment." - if command -v python3 >/dev/null; then - python3 -m venv "$DIR/venv" - elif command -v python3.10 >/dev/null; then - python3.10 -m venv "$DIR/venv" - else - echo "Valid python3 or python3.10 binary not found." - echo "Cannot proceed with the python steps." - return 1 - fi - - # Activate the virtual environment - source "$DIR/venv/bin/activate" - - # Updating pip if there is one - echo "Checking for pip updates before Python operations." - python3 -m pip install --upgrade pip >&3 - - echo "Installing python dependencies. This could take a few minutes as it downloads files." - echo "If this operation ever runs too long, you can rerun this script in verbose mode to check." - case "$OSTYPE" in - "linux-gnu"*) pip install torch==1.12.1+cu116 torchvision==0.13.1+cu116 \ - --extra-index-url https://download.pytorch.org/whl/cu116 >&3 && - pip install -U -I --no-deps \ - https://github.com/C43H66N12O12S2/stable-diffusion-webui/releases/downloadlinux/xformers-0.0.14.dev0-cp310-cp310-linux_x86_64.whl >&3 ;; - "darwin"*) pip install torch==2.0.0 torchvision==0.15.1 \ - -f https://download.pytorch.org/whl/cpu/torch_stable.html >&3 ;; - "cygwin") - : - ;; - "msys") - : - ;; - esac - - if [ "$RUNPOD" = true ]; then - echo "Installing tenssort." - pip install tensorrt >&3 - fi - - # DEBUG ONLY (Update this version number to whatever PyCharm recommends) - # pip install pydevd-pycharm~=223.8836.43 - - #This will copy our requirements.txt file out, make the khoya_ss lib a dynamic location then cleanup. - echo "Copying $DIR/requirements.txt to /tmp/requirements_tmp.txt" >&3 - echo "Replacing the . for lib to our DIR variable in tmp/requirements_tmp.txt." >&3 - awk -v dir="$DIR" '/#.*kohya_ss.*library/{print; getline; sub(/^\.$/, dir)}1' "$DIR/requirements.txt" >/tmp/requirements_tmp.txt - if [ $VERBOSITY == 2 ]; then - python -m pip install --quiet --use-pep517 --upgrade -r /tmp/requirements_tmp.txt >&3 - else - python -m pip install --use-pep517 --upgrade -r /tmp/requirements_tmp.txt >&3 - fi - - echo "Removing the temp requirements file." - if [ -f /tmp/requirements_tmp.txt ]; then - rm /tmp/requirements_tmp.txt - fi - - if [ -n "$VIRTUAL_ENV" ]; then - if command -v deactivate >/dev/null; then - echo "Exiting Python virtual environment." - deactivate - else - echo "deactivate command not found. Could still be in the Python virtual environment." - fi - fi -} - -# Attempt to non-interactively install a default accelerate config file unless specified otherwise. -# Documentation for order of precedence locations for configuration file for automated installation: -# https://huggingface.co/docs/accelerate/basic_tutorials/launch#custom-configurations -configure_accelerate() { - echo "Source accelerate config location: $DIR/config_files/accelerate/default_config.yaml" >&3 - if [ "$INTERACTIVE" = true ]; then - accelerate config - else - if env_var_exists HF_HOME; then - if [ ! -f "$HF_HOME/accelerate/default_config.yaml" ]; then - mkdir -p "$HF_HOME/accelerate/" && - echo "Target accelerate config location: $HF_HOME/accelerate/default_config.yaml" >&3 - cp "$DIR/config_files/accelerate/default_config.yaml" "$HF_HOME/accelerate/default_config.yaml" && - echo "Copied accelerate config file to: $HF_HOME/accelerate/default_config.yaml" - fi - elif env_var_exists XDG_CACHE_HOME; then - if [ ! -f "$XDG_CACHE_HOME/huggingface/accelerate" ]; then - mkdir -p "$XDG_CACHE_HOME/huggingface/accelerate" && - echo "Target accelerate config location: $XDG_CACHE_HOME/accelerate/default_config.yaml" >&3 - cp "$DIR/config_files/accelerate/default_config.yaml" "$XDG_CACHE_HOME/huggingface/accelerate/default_config.yaml" && - echo "Copied accelerate config file to: $XDG_CACHE_HOME/huggingface/accelerate/default_config.yaml" - fi - elif env_var_exists HOME; then - if [ ! -f "$HOME/.cache/huggingface/accelerate" ]; then - mkdir -p "$HOME/.cache/huggingface/accelerate" && - echo "Target accelerate config location: $HOME/accelerate/default_config.yaml" >&3 - cp "$DIR/config_files/accelerate/default_config.yaml" "$HOME/.cache/huggingface/accelerate/default_config.yaml" && - echo "Copying accelerate config file to: $HOME/.cache/huggingface/accelerate/default_config.yaml" - fi - else - echo "Could not place the accelerate configuration file. Please configure manually." - sleep 2 - accelerate config - fi - fi -} - -# Offer a warning and opportunity to cancel the installation if < 10Gb of Free Space detected -check_storage_space() { - if [ "$SKIP_SPACE_CHECK" = false ]; then - if [ "$(size_available)" -lt 10 ]; then - echo "You have less than 10Gb of free space. This installation may fail." - MSGTIMEOUT=10 # In seconds - MESSAGE="Continuing in..." - echo "Press control-c to cancel the installation." - for ((i = MSGTIMEOUT; i >= 0; i--)); do - printf "\r${MESSAGE} %ss. " "${i}" - sleep 1 - done - fi - fi -} - -# These are the git operations that will run to update or clone the repo -update_kohya_ss() { - if [ "$SKIP_GIT_UPDATE" = false ]; then - if command -v git >/dev/null; then - # First, we make sure there are no changes that need to be made in git, so no work is lost. - if [ "$(git -C "$DIR" status --porcelain=v1 2>/dev/null | wc -l)" -gt 0 ] && - echo "These files need to be committed or discarded: " >&4 && - git -C "$DIR" status >&4; then - echo "There are changes that need to be committed or discarded in the repo in $DIR." - echo "Commit those changes or run this script with -n to skip git operations entirely." - exit 1 - fi - - echo "Attempting to clone $GIT_REPO." - if [ ! -d "$DIR/.git" ]; then - echo "Cloning and switching to $GIT_REPO:$BRANCH" >*4 - git -C "$DIR" clone -b "$BRANCH" "$GIT_REPO" "$(basename "$DIR")" >&3 - git -C "$DIR" switch "$BRANCH" >&4 - else - echo "git repo detected. Attempting to update repository instead." - echo "Updating: $GIT_REPO" - git -C "$DIR" pull "$GIT_REPO" "$BRANCH" >&3 - if ! git -C "$DIR" switch "$BRANCH" >&4; then - echo "Branch $BRANCH did not exist. Creating it." >&4 - git -C "$DIR" switch -c "$BRANCH" >&4 - fi - fi - else - echo "You need to install git." - echo "Rerun this after installing git or run this script with -n to skip the git operations." - fi - else - echo "Skipping git operations." - fi -} - -# Start OS-specific detection and work -if [[ "$OSTYPE" == "linux-gnu"* ]]; then - # Check if root or sudo - root=false - if [ "$EUID" = 0 ]; then - root=true - elif command -v id >/dev/null && [ "$(id -u)" = 0 ]; then - root=true - elif [ "$UID" = 0 ]; then - root=true - fi - - get_distro_name() { - local line - if [ -f /etc/os-release ]; then - # We search for the line starting with ID= - # Then we remove the ID= prefix to get the name itself - line="$(grep -Ei '^ID=' /etc/os-release)" - echo "Raw detected os-release distro line: $line" >&5 - line=${line##*=} - echo "$line" - return 0 - elif command -v python >/dev/null; then - line="$(python -mplatform)" - echo "$line" - return 0 - elif command -v python3 >/dev/null; then - line="$(python3 -mplatform)" - echo "$line" - return 0 - else - line="None" - echo "$line" - return 1 - fi - } - - # We search for the line starting with ID_LIKE= - # Then we remove the ID_LIKE= prefix to get the name itself - # This is the "type" of distro. For example, Ubuntu returns "debian". - get_distro_family() { - local line - if [ -f /etc/os-release ]; then - if grep -Eiq '^ID_LIKE=' /etc/os-release >/dev/null; then - line="$(grep -Ei '^ID_LIKE=' /etc/os-release)" - echo "Raw detected os-release distro family line: $line" >&5 - line=${line##*=} - echo "$line" - return 0 - else - line="None" - echo "$line" - return 1 - fi - else - line="None" - echo "$line" - return 1 - fi - } - - check_storage_space - - # This is the pre-install work for a kohya installation on a runpod - if [ "$RUNPOD" = true ]; then - if [ -d "$VENV_DIR" ]; then - echo "Pre-existing installation on a runpod detected." - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:"$VENV_DIR"/lib/python3.10/site-packages/tensorrt/ - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:"$VENV_DIR"/lib/python3.10/site-packages/nvidia/cuda_runtime/lib/ - cd "$DIR" || exit 1 - fi - fi - - update_kohya_ss - - distro=get_distro_name - family=get_distro_family - echo "Raw detected distro string: $distro" >&4 - echo "Raw detected distro family string: $family" >&4 - - echo "Installing Python TK if not found on the system." - if "$distro" | grep -qi "Ubuntu" || "$family" | grep -qi "Ubuntu"; then - echo "Ubuntu detected." - if [ $(dpkg-query -W -f='${Status}' python3-tk 2>/dev/null | grep -c "ok installed") = 0 ]; then - if [ "$root" = true ]; then - apt update -y >&3 && apt install -y python3-tk >&3 - else - echo "This script needs to be run as root or via sudo to install packages." - exit 1 - fi - else - echo "Python TK found! Skipping install!" - fi - elif "$distro" | grep -Eqi "Fedora|CentOS|Redhat"; then - echo "Redhat or Redhat base detected." - if ! rpm -qa | grep -qi python3-tkinter; then - if [ "$root" = true ]; then - dnf install python3-tkinter -y >&3 - else - echo "This script needs to be run as root or via sudo to install packages." - exit 1 - fi - fi - elif "$distro" | grep -Eqi "arch" || "$family" | grep -qi "arch"; then - echo "Arch Linux or Arch base detected." - if ! pacman -Qi tk >/dev/null; then - if [ "$root" = true ]; then - pacman --noconfirm -S tk >&3 - else - echo "This script needs to be run as root or via sudo to install packages." - exit 1 - fi - fi - elif "$distro" | grep -Eqi "opensuse" || "$family" | grep -qi "opensuse"; then - echo "OpenSUSE detected." - if ! rpm -qa | grep -qi python-tk; then - if [ "$root" = true ]; then - zypper install -y python-tk >&3 - else - echo "This script needs to be run as root or via sudo to install packages." - exit 1 - fi - fi - elif [ "$distro" = "None" ] || [ "$family" = "None" ]; then - if [ "$distro" = "None" ]; then - echo "We could not detect your distribution of Linux. Please file a bug report on github with the contents of your /etc/os-release file." - fi - - if [ "$family" = "None" ]; then - echo "We could not detect the family of your Linux distribution. Please file a bug report on github with the contents of your /etc/os-release file." - fi - fi - - install_python_dependencies - - # We need just a little bit more setup for non-interactive environments - if [ "$RUNPOD" = true ]; then - # Symlink paths - libnvinfer_plugin_symlink="$VENV_DIR/lib/python3.10/site-packages/tensorrt/libnvinfer_plugin.so.7" - libnvinfer_symlink="$VENV_DIR/lib/python3.10/site-packages/tensorrt/libnvinfer.so.7" - libcudart_symlink="$VENV_DIR/lib/python3.10/site-packages/nvidia/cuda_runtime/lib/libcudart.so.11.0" - - #Target file paths - libnvinfer_plugin_target="$VENV_DIR/lib/python3.10/site-packages/tensorrt/libnvinfer_plugin.so.8" - libnvinfer_target="$VENV_DIR/lib/python3.10/site-packages/tensorrt/libnvinfer.so.8" - libcudart_target="$VENV_DIR/lib/python3.10/site-packages/nvidia/cuda_runtime/lib/libcudart.so.12" - - echo "Checking symlinks now." - create_symlinks "$libnvinfer_plugin_symlink" "$libnvinfer_plugin_target" - create_symlinks "$libnvinfer_symlink" "$libnvinfer_target" - create_symlinks "$libcudart_symlink" "$libcudart_target" - - export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$VENV_DIR/lib/python3.10/site-packages/tensorrt/" - export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$VENV_DIR/lib/python3.10/site-packages/nvidia/cuda_runtime/lib/" - - configure_accelerate - - # This is a non-interactive environment, so just directly call gui.sh after all setup steps are complete. - if command -v bash >/dev/null; then - if [ "$PUBLIC" = false ]; then - bash "$DIR"/gui.sh - else - bash "$DIR"/gui.sh --share - fi - else - # This shouldn't happen, but we're going to try to help. - if [ "$PUBLIC" = false ]; then - sh "$DIR"/gui.sh - else - sh "$DIR"/gui.sh --share - fi - fi - fi - - echo -e "Setup finished! Run \e[0;92m./gui.sh\e[0m to start." - echo "Please note if you'd like to expose your public server you need to run ./gui.sh --share" -elif [[ "$OSTYPE" == "darwin"* ]]; then - # The initial setup script to prep the environment on macOS - # xformers has been omitted as that is for Nvidia GPUs only - - if ! command -v brew >/dev/null; then - echo "Please install homebrew first. This is a requirement for the remaining setup." - echo "You can find that here: https://brew.sh" - #shellcheck disable=SC2016 - echo 'The "brew" command should be in $PATH to be detected.' - exit 1 - fi - - check_storage_space - - # Install base python packages - echo "Installing Python 3.10 if not found." - if ! brew ls --versions python@3.10 >/dev/null; then - echo "Installing Python 3.10." - brew install python@3.10 >&3 - else - echo "Python 3.10 found!" - fi - echo "Installing Python-TK 3.10 if not found." - if ! brew ls --versions python-tk@3.10 >/dev/null; then - echo "Installing Python TK 3.10." - brew install python-tk@3.10 >&3 - else - echo "Python Tkinter 3.10 found!" - fi - - update_kohya_ss - - if ! install_python_dependencies; then - echo "You may need to install Python. The command for this is brew install python@3.10." - fi - - configure_accelerate - echo -e "Setup finished! Run ./gui.sh to start." -elif [[ "$OSTYPE" == "cygwin" ]]; then - # Cygwin is a standalone suite of Linux utilies on Windows - echo "This hasn't been validated on cygwin yet." -elif [[ "$OSTYPE" == "msys" ]]; then - # MinGW has the msys environment which is a standalone suite of Linux utilies on Windows - # "git bash" on Windows may also be detected as msys. - echo "This hasn't been validated in msys (mingw) on Windows yet." -fi diff --git a/ubuntu_setup.sh b/ubuntu_setup.sh new file mode 100755 index 0000000..1431155 --- /dev/null +++ b/ubuntu_setup.sh @@ -0,0 +1,12 @@ +#!/bin/bash +echo installing tk +sudo apt install python3-tk +python3 -m venv venv +source venv/bin/activate +pip install torch==1.12.1+cu116 torchvision==0.13.1+cu116 --extra-index-url https://download.pytorch.org/whl/cu116 +pip install --use-pep517 --upgrade -r requirements.txt +pip install -U -I --no-deps https://github.com/C43H66N12O12S2/stable-diffusion-webui/releases/download/linux/xformers-0.0.14.dev0-cp310-cp310-linux_x86_64.whl + +accelerate config + +echo -e "setup finished! run \e[0;92m./gui.sh\e[0m to start" diff --git a/upgrade.sh b/upgrade.sh new file mode 100755 index 0000000..f01e7b7 --- /dev/null +++ b/upgrade.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Check if there are any changes that need to be committed +if [[ -n $(git status --short) ]]; then + echo "There are changes that need to be committed. Please stash or undo your changes before running this script." >&2 + exit 1 +fi + +# Pull the latest changes from the remote repository +git pull + +# Activate the virtual environment +source venv/bin/activate + +# Upgrade the required packages +pip install --upgrade -r requirements.txt diff --git a/upgrade_macos.sh b/upgrade_macos.sh new file mode 100755 index 0000000..2e26c55 --- /dev/null +++ b/upgrade_macos.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Check if there are any changes that need to be committed +if [[ -n $(git status --short) ]]; then + echo "There are changes that need to be committed. Please stash or undo your changes before running this script." >&2 + exit 1 +fi + +# Pull the latest changes from the remote repository +git pull + +# Activate the virtual environment +source venv/bin/activate + +# Upgrade the required packages +pip install --upgrade -r requirements_macos.txt