Skip to content

Commit

Permalink
fix: rollback old commit
Browse files Browse the repository at this point in the history
  • Loading branch information
@Linaqruf
Linaqruf committed Jan 8, 2023
1 parent 7e9fc72 commit aac6765
Showing 1 changed file with 80 additions and 165 deletions.
245 changes: 80 additions & 165 deletions kohya-trainer.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@
"id": "slgjeYgd6pWp"
},
"source": [
"# Kohya LoRA Fine-Tuning\n"
"# Kohya Trainer V10 - VRAM 12GB\n",
"### The Best Way for People Without Good GPUs to Fine-Tune the Stable Diffusion Model"
]
},
{
Expand Down Expand Up @@ -88,7 +89,6 @@
" #@markdown This will install required Python packages\n",
" !pip install --upgrade -r requirements.txt\n",
" !pip install -U gallery-dl\n",
" !pip install imjoy-elfinder\n",
"\n",
" if Install_xformers:\n",
" !pip install -U -I --no-deps https://github.com/camenduru/stable-diffusion-webui-colab/releases/download/0.0.15/xformers-0.0.15.dev0+189828c.d20221207-cp38-cp38-linux_x86_64.whl\n",
Expand All @@ -108,35 +108,6 @@
"write_basic_config(save_location = accelerate_config) # Write a config file"
]
},
{
"cell_type": "code",
"source": [
"#@title Install Special File Explorer for Colab\n",
"#@markdown this will work real-time even though you're running other cells\n",
"import threading\n",
"from google.colab import output\n",
"from imjoy_elfinder.app import main\n",
"\n",
"# start imjoy-elfinder server\n",
"thread = threading.Thread(target=main, args=[[\"--root-dir=/content\", \"--port=8765\"]])\n",
"thread.start()\n",
"\n",
"open_in_new_tab = True #@param {type:\"boolean\"}\n",
"\n",
"if open_in_new_tab:\n",
" # open imjoy-elfinder in a new tab\n",
" output.serve_kernel_port_as_window(8765)\n",
"else:\n",
" # view the \n",
" output.serve_kernel_port_as_iframe(8765, height='500')\n"
],
"metadata": {
"cellView": "form",
"id": "t5kXnw1G9teH"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
Expand Down Expand Up @@ -308,8 +279,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "Kt1GzntK_apb",
"cellView": "form"
"id": "Kt1GzntK_apb"
},
"outputs": [],
"source": [
Expand All @@ -323,7 +293,7 @@
"# Set configuration options\n",
"train_data_dir = \"/content/fine_tune/train_data\" #@param {'type' : 'string'}\n",
"booru = \"Gelbooru\" #@param [\"\", \"Danbooru\", \"Gelbooru\"]\n",
"tag1 = \"masabodo\" #@param {type: \"string\"}\n",
"tag1 = \"hito_komoru\" #@param {type: \"string\"}\n",
"tag2 = \"\" #@param {type: \"string\"}\n",
"download_tags = False #@param {type: \"boolean\"}\n",
"# Construct the search query\n",
Expand Down Expand Up @@ -489,7 +459,7 @@
"%cd /content/cafe-aesthetic-scorer/\n",
"#@markdown If you want to append custom tag, you can do that here. This cell will add custom tag at the beginning of lines\n",
"train_data_dir = \"/content/fine_tune/train_data\" #@param {type:\"string\"}\n",
"custom_tag = \"fumo\" #@param {type:\"string\"}\n",
"custom_tag = \"fum\" #@param {type:\"string\"}\n",
"caption_extension = \"txt\" #@param [\"txt\",\"caption\"]\n",
"#@markdown Tick this if you want to append custom tag at the end of lines instead\n",
"append = False #@param {type: \"boolean\"}\n",
Expand Down Expand Up @@ -803,23 +773,20 @@
"source": [
"#@title Define Important folder\n",
"import os\n",
"V2 = \"none\" #@param [\"none\", \"V2_base\", \"V2_768_v\"] {allow-input: false}\n",
"\n",
"pre_trained_model_path =\"/content/pre_trained_model/Anything-V3.0-pruned.ckpt\" #@param {'type':'string'}\n",
"meta_lat_json_dir = \"/content/fine_tune/meta_lat.json\" #@param {'type':'string'}\n",
"train_data_dir = \"/content/fine_tune/train_data\" #@param {'type':'string'}\n",
"output_dir =\"/content/fine_tune/output\" #@param {'type':'string'}\n",
"\n",
"# Check if directory exists\n",
"if not os.path.exists(output_dir):\n",
" # Create directory if it doesn't exist\n",
" os.makedirs(output_dir)\n",
"resume_path = \"\" #@param {'type':'string'}\n",
"\n",
"# List of important folder paths\n",
"folder_paths = [\n",
" pre_trained_model_path,\n",
" meta_lat_json_dir,\n",
" train_data_dir,\n",
" output_dir,\n",
" resume_path\n",
"]\n",
"\n",
"# Check if each folder exists\n",
Expand All @@ -833,29 +800,7 @@
" except:\n",
" print(f'An error occurred while checking if {folder_path} exists')\n",
" else:\n",
" print('Empty folder path')\n",
"\n",
"inference_url = \"https://raw.githubusercontent.com/Stability-AI/stablediffusion/main/configs/stable-diffusion/\"\n",
"\n",
"if V2 == \"V2_base\":\n",
" v2_model = \"--v2\"\n",
" v2_768v_model= \"\"\n",
" inference_url += \"v2-inference.yaml\"\n",
"elif V2 == \"V2_768_v\":\n",
" v2_model = \"--v2\"\n",
" v2_768v_model = \"--v2_parameterization\"\n",
" inference_url += \"v2-inference-v.yaml\"\n",
"else:\n",
" v2_model = \"\"\n",
" v2_768v_model = \"\"\n",
"\n",
"try:\n",
" if V2 != \"none\":\n",
" !wget {inference_url} -O {output_dir}/last.yaml\n",
" print(\"File successfully downloaded\")\n",
"except:\n",
" print(\"There was an error downloading the file. Please check the URL and try again.\")\n",
"\n"
" print('Empty folder path')\n"
],
"metadata": {
"cellView": "form",
Expand All @@ -864,40 +809,6 @@
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"#@title Define Specific LoRA Training parameter\n",
"#@markdown ## LoRA - Low Rank Adaptation Fine-Tuning\n",
"network_dim = 128 #@param {'type':'number'}\n",
"network_module = \"network.lora\" #@param {'type':'string'}\n",
"#@markdown `Specify network_weights for resume training`\n",
"network_weights = \"\" #@param {'type':'string'}\n",
"network_train_on = \"all\" #@param ['all','unet_only', 'text_encoder_only'] {'type':'string'}\n",
"#@markdown When neither `--network_train_unet_only` nor `--network_train_text_encoder_only` is specified (default), both Text Encoder and U-Net LoRA modules are enabled.\n",
"\n",
"unet_lr = 0 #@param {'type':'number'}\n",
"text_encoder_lr = 0 #@param {'type':'number'}\n",
"#@markdown Leave the value to 0 (zero) to use default learning rate\n",
"if network_weights == \"\":\n",
" network_weights_value =\"\"\n",
"else:\n",
" network_weights_value =\"--network_weights \" + str(network_weights)\n",
"\n",
"if network_train_on == \"unet_only\":\n",
" lora_module_to_train = \"--network_train_unet_only\"\n",
"elif network_train_on == \"text_encoder_only\":\n",
" lora_module_to_train = \"--network_train_text_encoder_only\"\n",
"else:\n",
" lora_module_to_train = \"\""
],
"metadata": {
"cellView": "form",
"id": "32aNPjnYAcu_"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"execution_count": null,
Expand All @@ -909,114 +820,128 @@
"source": [
"#@title Training begin\n",
"#@markdown ### Define Parameters\n",
"import glob \n",
"import math\n",
"\n",
"\n",
"V2 = \"none\" #@param [\"none\", \"V2_base\", \"V2_768_v\"] {allow-input: false}\n",
"num_cpu_threads_per_process = 8 #@param {'type':'integer'}\n",
"train_batch_size = 4 #@param {type: \"slider\", min: 1, max: 10}\n",
"learning_rate =\"1e-4\" #@param {'type':'string'}\n",
"num_epoch = 100 #@param {'type':'integer'}\n",
"dataset_repeats = 1 #@param {'type':'integer'}\n",
"resolution = 512 #@param {'type':'integer'}\n",
"save_state = True #@param {'type':'boolean'}\n",
"train_batch_size = 1 #@param {type: \"slider\", min: 1, max: 10}\n",
"learning_rate =\"2e-6\" #@param {'type':'string'}\n",
"max_train_steps = 5000 #@param {'type':'integer'}\n",
"train_text_encoder = False #@param {'type':'boolean'}\n",
"lr_scheduler = \"constant\" #@param [\"linear\", \"cosine\", \"cosine_with_restarts\", \"polynomial\", \"constant\", \"constant_with_warmup\"] {allow-input: false}\n",
"max_token_length = \"225\" #@param [\"150\", \"225\"] {allow-input: false}\n",
"clip_skip = 2 #@param {type: \"slider\", min: 1, max: 10}\n",
"mixed_precision = \"fp16\" #@param [\"no\",\"fp16\",\"bf16\"] {allow-input: false}\n",
"save_model_as = 'safetensors' #@param ['default', 'ckpt', 'pt', 'safetensors'] {'type':'string'}\n",
"save_precision = 'fp16' #@param ['none','float', 'fp16', 'bf16'] {'type':'string'}\n",
"save_every_n_epochs = 1 #@param {'type':'integer'}\n",
"save_model_as = \"ckpt\" #@param [\"default\", \"ckpt\", \"safetensors\", \"diffusers\", \"diffusers_safetensors\"] {allow-input: false}\n",
"save_precision = \"None\" #@param [\"None\",\"float\", \"fp16\", \"bf16\"] {allow-input: false}\n",
"save_every_n_epochs = 50 #@param {'type':'integer'}\n",
"gradient_accumulation_steps = 1 #@param {type: \"slider\", min: 1, max: 10}\n",
"#@markdown ### Log And Debug\n",
"log_prefix = \"LoRA-fine-tune-style1\" #@param {'type':'string'}\n",
"log_prefix = \"fine-tune-style1\" #@param {'type':'string'}\n",
"logs_dst = \"/content/fine_tune/training_logs\" #@param {'type':'string'}\n",
"debug_mode = False #@param {'type':'boolean'}\n",
"\n",
"# Hidden Config\n",
"save_state = False\n",
"resume_path = \"\"\n",
"#V2 Inference\n",
"\n",
"# Check if directory exists\n",
"if not os.path.exists(output_dir):\n",
" # Create directory if it doesn't exist\n",
" os.makedirs(output_dir)\n",
"\n",
"inference_url = \"https://raw.githubusercontent.com/Stability-AI/stablediffusion/main/configs/stable-diffusion/\"\n",
"\n",
"if V2 == \"V2_base\":\n",
" v2_model = \"--v2\"\n",
" v2_768v_model= \"\"\n",
" inference_url += \"v2-inference.yaml\"\n",
"elif V2 == \"V2_768_v\":\n",
" v2_model = \"--v2\"\n",
" v2_768v_model = \"--v2_parameterization\"\n",
" inference_url += \"v2-inference-v.yaml\"\n",
"else:\n",
" v2_model = \"\"\n",
" v2_768v_model = \"\"\n",
"\n",
"try:\n",
" if V2 != \"none\":\n",
" !wget {inference_url} -O {output_dir}/last.yaml\n",
" print(\"File successfully downloaded\")\n",
"except:\n",
" print(\"There was an error downloading the file. Please check the URL and try again.\")\n",
"\n",
"if V2 == \"none\":\n",
" penultimate_layer = \"--clip_skip\" + \"=\" + \"{}\".format(clip_skip)\n",
"else:\n",
" penultimate_layer = \"\"\n",
"\n",
"save_model_as_value_mapping = {\n",
" \"default\": \"\",\n",
" \"ckpt\": \"--save_model_as=ckpt\",\n",
" \"pt\": \"--save_model_as=pt\",\n",
" \"safetensors\": \"--save_model_as=safetensors\"\n",
"}\n",
"save_model_as_value = save_model_as_value_mapping[save_model_as]\n",
"if save_model_as == \"default\":\n",
" sv_model = \"\"\n",
"else: \n",
" sv_model = \"--save_model_as \" + str(save_model_as)\n",
"\n",
"save_state_value_mapping = {True: \"--save_state\", False: \"\"}\n",
"save_state_value = save_state_value_mapping[save_state]\n",
"if save_state == True:\n",
" sv_state = \"--save_state\"\n",
"else:\n",
" sv_state = \"\"\n",
"\n",
"if resume_path == \"\":\n",
" resume_value = \"\"\n",
" rs_state = \"\"\n",
"else:\n",
" resume_value = \"--resume \" + str(resume_path)\n",
" rs_state = \"--resume \" + str(resume_path)\n",
"\n",
"save_precision_value_mapping = {\n",
" \"none\": \"\",\n",
" \"float\": \"--save_precision=float\",\n",
" \"fp16\": \"--save_precision=fp16\",\n",
" \"bf16\": \"--save_precision=bf16\"\n",
"}\n",
"save_precision_value = save_precision_value_mapping[save_precision]\n",
"if save_every_n_epochs == 0 :\n",
" save_epoch = \"\"\n",
"else:\n",
" save_epoch = \"--save_every_n_epochs\" + \"=\" + \"{}\".format(save_every_n_epochs)\n",
"\n",
"debug_mode_value_mapping = {True: \"--debug\", False: \"\"}\n",
"debug_mode_value = debug_mode_value_mapping[debug_mode]\n",
"if save_precision == \"None\":\n",
" sv_precision = \"\"\n",
"else :\n",
" sv_precision = \"--save_precision=\" + str(save_precision)\n",
"\n",
"# Get number of valid images\n",
"image_num = len(glob.glob(train_data_dir + \"/*.npz\"))\n",
"if debug_mode == True:\n",
" debug_dataset = \"--debug_dataset\"\n",
"else:\n",
" debug_dataset = \"\"\n",
"\n",
"print(\"Total Train Data =\", image_num)\n",
"print(\"Total Epoch=\", num_epoch)\n",
"print(\"Dataset repeats =\", dataset_repeats, \"x\")\n",
"repeats = image_num * dataset_repeats\n",
"print(\"Total Repeats =\", image_num, \"*\", dataset_repeats, \"=\", repeats)\n",
"if train_text_encoder == True:\n",
" text_encoder = \"--train_text_encoder\"\n",
"else:\n",
" text_encoder = \"\"\n",
"\n",
"# calculate max_train_steps\n",
"max_train_steps = math.ceil(repeats / train_batch_size * num_epoch)\n",
"print(\"max_train_steps =\", repeats, \"/\", train_batch_size, \"*\", num_epoch ,\"=\", max_train_steps, \"\\n\")\n",
"\n",
"%cd /content/kohya-trainer\n",
"\n",
"!accelerate launch \\\n",
" --config_file {accelerate_config} \\\n",
" --num_cpu_threads_per_process {num_cpu_threads_per_process} \\\n",
" train_network.py \\\n",
" fine_tune.py \\\n",
" {v2_model} \\\n",
" {v2_768v_model} \\\n",
" --network_module=networks.lora \\\n",
" --network_dim {network_dim} \\\n",
" {network_weights_value} \\\n",
" {lora_module_to_train} \\\n",
" --pretrained_model_name_or_path={pre_trained_model_path} \\\n",
" --in_json {meta_lat_json_dir} \\\n",
" --train_data_dir={train_data_dir} \\\n",
" --output_dir={output_dir} \\\n",
" --resolution {resolution} \\\n",
" --shuffle_caption \\\n",
" --train_batch_size={train_batch_size} \\\n",
" --learning_rate={learning_rate} \\\n",
" --lr_scheduler={lr_scheduler} \\\n",
" --max_token_length={max_token_length} \\\n",
" {penultimate_layer} \\\n",
" --mixed_precision={mixed_precision} \\\n",
" --dataset_repeats {dataset_repeats} \\\n",
" --max_train_steps={max_train_steps} \\\n",
" --use_8bit_adam \\\n",
" --xformers \\\n",
" --gradient_checkpointing \\\n",
" --gradient_accumulation_steps {gradient_accumulation_steps} \\\n",
" {save_model_as_value} \\\n",
" {save_state_value} \\\n",
" {resume_value} \\\n",
" --save_every_n_epochs {save_every_n_epochs} \\\n",
" {save_precision_value} \\\n",
" {debug_mode_value} \\\n",
" {sv_model} \\\n",
" {text_encoder} \\\n",
" {sv_state} \\\n",
" {rs_state} \\\n",
" {save_epoch} \\\n",
" {sv_precision} \\\n",
" {debug_dataset} \\\n",
" --logging_dir={logs_dst} \\\n",
" --log_prefix {log_prefix}\n",
"\n"
Expand All @@ -1035,12 +960,6 @@
"cell_type": "code",
"source": [
"#@title Inference\n",
"#@markdown LoRA Config\n",
"network_dim = 128 #@param {'type':'number'}\n",
"network_weights = \"/content/tmp/epoch-000034.safetensors\" #@param {'type':'string'}\n",
"network_mul = 1.0 #@param {'type':'number'}\n",
"#@markdown `Choose range from 0 to 1.0`\n",
"#@markdown Other Config\n",
"V2 = \"none\" #@param [\"none\", \"V2_base\", \"V2_768_v\"] {allow-input: false}\n",
"prompt = \"masterpiece, best quality, 1girl, aqua eyes, baseball cap, blonde hair, closed mouth, earrings, green background, hat, hoop earrings, jewelry, looking at viewer, shirt, short hair, simple background, solo, upper body, yellow shirt\" #@param {type: \"string\"}\n",
"negative = \"worst quality, low quality, medium quality, deleted, lowres, comic, bad anatomy, bad hands, text, error, missing fingers, extra digit, fewer digits, cropped, jpeg artifacts, signature, watermark, username, blurry\" #@param {type: \"string\"}\n",
Expand Down Expand Up @@ -1102,10 +1021,6 @@
" --batch_size {batch_size} \\\n",
" --images_per_prompt {images_per_prompt} \\\n",
" {penultimate_layer} \\\n",
" --network_module=networks.lora \\\n",
" --network_weight=\"{network_weights}\" \\\n",
" --network_mul 1.0 \\\n",
" --network_dim {network_dim} \\\n",
" --prompt \"{prompt} --n {negative}\"\n",
"\n"
],
Expand Down

0 comments on commit aac6765

Please sign in to comment.