From 357ec81a1719d24f142108ad156a0585da327a66 Mon Sep 17 00:00:00 2001 From: Eric Wang Date: Mon, 13 Mar 2023 17:23:29 -0700 Subject: [PATCH] decapoda --- README.md | 31 +++++--------- finetune.py | 40 +++++------------- generate.py | 15 ++++--- loss.ipynb | 119 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 148 insertions(+), 57 deletions(-) create mode 100644 loss.ipynb diff --git a/README.md b/README.md index 56625384..1b73db2c 100644 --- a/README.md +++ b/README.md @@ -1,34 +1,25 @@ -# alpaca-lora +# alpaca-lora (WIP) -This repository contains code for reproducing the Stanford Alpaca results. Users will need to have LLaMA weights on hand and be ready to fork `transformers`. +This repository contains code for reproducing the [Stanford Alpaca results](https://github.com/tatsu-lab/stanford_alpaca#data-release). Users will need to be ready to fork `transformers`. +# Setup -1. Install dependencies +1. Install dependencies (**install zphang's transformers fork**) ``` -pip install -q bitsandbytes datasets accelerate loralib +pip install -q datasets accelerate loralib sentencepiece pip install -q git+https://github.com/zphang/transformers@llama_push -pip install -q git+https://github.com/huggingface/peft.git\ +pip install -q git+https://github.com/huggingface/peft.git ``` -2. Convert weights +2. [Install bitsandbytes from source](https://github.com/TimDettmers/bitsandbytes/blob/main/compile_from_source.md) -``` -python conversion.py --input_dir [LLAMA_DIR]/LLaMA --model_size 7B --output_dir ./7B -``` -3. Modify hyperparams in `finetune.py` +# Inference -``` -MICRO_BATCH_SIZE = 12 -BATCH_SIZE = 36 -EPOCHS = 3 -LEARNING_RATE = 2e-5 -``` +See `generate.py`. -4. Run experiments +# Training -``` -python finetune.py -``` \ No newline at end of file +Under construction. \ No newline at end of file diff --git a/finetune.py b/finetune.py index a498ed73..9a0e9b0a 100644 --- a/finetune.py +++ b/finetune.py @@ -10,32 +10,14 @@ from peft import prepare_model_for_int8_training, LoraConfig, get_peft_model model = LLaMAForCausalLM.from_pretrained( - "./7B/llama-7b", + "decapoda-research/llama-7b-hf", load_in_8bit=True, - max_sequence_length=128, # data length device_map="auto", ) -tokenizer = LLaMATokenizer.from_pretrained("./7B/tokenizer") +tokenizer = LLaMATokenizer.from_pretrained("decapoda-research/llama-7b-hf") - -def print_trainable_parameters(model): - """ - Prints the number of trainable parameters in the model. - """ - trainable_params = 0 - all_param = 0 - for _, param in model.named_parameters(): - all_param += param.numel() - if param.requires_grad: - trainable_params += param.numel() - print( - f"trainable params: {trainable_params} || all params: {all_param} || trainable%: {100 * trainable_params / all_param}" - ) - - -print_trainable_parameters(model) model = prepare_model_for_int8_training(model) config = LoraConfig( @@ -48,8 +30,6 @@ def print_trainable_parameters(model): ) model = get_peft_model(model, config) -print_trainable_parameters(model) - tokenizer.pad_token = tokenizer.eos_token tokenizer.pad_token_id = tokenizer.eos_token_id @@ -77,21 +57,23 @@ def generate_prompt(data_point): ### Response:""" +# optimized for RTX 4090. +MICRO_BATCH_SIZE = 12 +BATCH_SIZE = 36 +GRADIENT_ACCUMULATION_STEPS = BATCH_SIZE // MICRO_BATCH_SIZE +EPOCHS = 1 +LEARNING_RATE = 2e-5 +CUTOFF_LEN = 128 + data = data.map( lambda data_point: tokenizer( generate_prompt(data_point), truncation=True, - max_length=128, + max_length=CUTOFF_LEN, padding="max_length", ) ) -MICRO_BATCH_SIZE = 12 -BATCH_SIZE = 36 -GRADIENT_ACCUMULATION_STEPS = BATCH_SIZE // MICRO_BATCH_SIZE -EPOCHS = 3 -LEARNING_RATE = 2e-5 - trainer = transformers.Trainer( model=model, diff --git a/generate.py b/generate.py index c786aa29..c767e363 100644 --- a/generate.py +++ b/generate.py @@ -1,23 +1,22 @@ -import torch -from peft import get_peft_model, PeftConfig, LoraConfig, PeftModel +from peft import PeftModel from transformers import LLaMATokenizer, LLaMAForCausalLM -tokenizer = LLaMATokenizer.from_pretrained("./7B/tokenizer") +tokenizer = LLaMATokenizer.from_pretrained("decapoda-research/llama-7b-hf") model = LLaMAForCausalLM.from_pretrained( - "./7B/llama-7b", + "decapoda-research/llama-7b-hf", load_in_8bit=True, device_map="auto", ) -model = PeftModel.from_pretrained(model, "./outputs") +model = PeftModel.from_pretrained(model, "tloen/alpaca-lora-7b") PROMPT = """Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request. ### Instruction: -Sort the following numbers. +Write a poem about the following topic. ### Input: -5, 2, 3 +Cars ### Response:""" @@ -26,7 +25,7 @@ return_tensors="pt", ) generation_output = model.generate( - **inputs, return_dict_in_generate=True, output_scores=True, max_new_tokens=50 + **inputs, return_dict_in_generate=True, output_scores=True, max_new_tokens=128 ) for s in generation_output.sequences: print(tokenizer.decode(s)) diff --git a/loss.ipynb b/loss.ipynb new file mode 100644 index 00000000..fcd48711 --- /dev/null +++ b/loss.ipynb @@ -0,0 +1,119 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "with open('outputs/checkpoint-4500/trainer_state.json') as f:\n", + " data = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Looking in indexes: https://pypi.org/simple, https://pypi.ngc.nvidia.com\n", + "Requirement already satisfied: matplotlib in /home/eric/miniconda3/envs/dl3/lib/python3.10/site-packages (3.7.1)\n", + "Requirement already satisfied: kiwisolver>=1.0.1 in /home/eric/miniconda3/envs/dl3/lib/python3.10/site-packages (from matplotlib) (1.4.4)\n", + "Requirement already satisfied: contourpy>=1.0.1 in /home/eric/miniconda3/envs/dl3/lib/python3.10/site-packages (from matplotlib) (1.0.7)\n", + "Requirement already satisfied: fonttools>=4.22.0 in /home/eric/miniconda3/envs/dl3/lib/python3.10/site-packages (from matplotlib) (4.39.0)\n", + "Requirement already satisfied: python-dateutil>=2.7 in /home/eric/miniconda3/envs/dl3/lib/python3.10/site-packages (from matplotlib) (2.8.2)\n", + "Requirement already satisfied: cycler>=0.10 in /home/eric/miniconda3/envs/dl3/lib/python3.10/site-packages (from matplotlib) (0.11.0)\n", + "Requirement already satisfied: numpy>=1.20 in /home/eric/miniconda3/envs/dl3/lib/python3.10/site-packages (from matplotlib) (1.23.5)\n", + "Requirement already satisfied: packaging>=20.0 in /home/eric/miniconda3/envs/dl3/lib/python3.10/site-packages (from matplotlib) (22.0)\n", + "Requirement already satisfied: pillow>=6.2.0 in /home/eric/miniconda3/envs/dl3/lib/python3.10/site-packages (from matplotlib) (9.3.0)\n", + "Requirement already satisfied: pyparsing>=2.3.1 in /home/eric/miniconda3/envs/dl3/lib/python3.10/site-packages (from matplotlib) (3.0.9)\n", + "Requirement already satisfied: six>=1.5 in /home/eric/miniconda3/envs/dl3/lib/python3.10/site-packages (from python-dateutil>=2.7->matplotlib) (1.16.0)\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "pip install matplotlib" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "from matplotlib import pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Text(0.5, 1.0, 'Loss')" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGzCAYAAAD9pBdvAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABO4UlEQVR4nO3dd1xT5x4G8CcDAihDHCiKYt0TUavFVQeuWmq9dtqp17bW2tbaaqWtq9Vq22u31i61U7tcrRMHUEdVEFQc4AYVcFSmMnPuHyThZCeMc8A838/Hj5CcJG8S4Dx5x+9VCIIggIiIiEgmSrkbQERERK6NYYSIiIhkxTBCREREsmIYISIiIlkxjBAREZGsGEaIiIhIVgwjREREJCuGESIiIpIVwwgRERHJimGEiIiIZMUwQkSVsnLlSigUCsTFxcndFCKqpRhGiIiISFYMI0RERCQrhhEiqnYJCQkYOXIkfHx8ULduXQwZMgT//POP0THFxcWYN28e2rRpAw8PD9SvXx/9+vVDVFSU4ZiMjAyMHz8ezZo1g0ajQZMmTTB69GicP39e4mdERFVJLXcDiOj2duzYMfTv3x8+Pj6YMWMG3Nzc8OWXX2LgwIGIiYlB7969AQBz587FwoULMXHiRPTq1Qs5OTmIi4vDoUOHMHToUADA2LFjcezYMbz44osIDg7GlStXEBUVhdTUVAQHB8v4LImoMhSCIAhyN4KIaq+VK1di/PjxOHjwIHr27Gl2/ZgxY7Bp0yacOHECd9xxBwAgPT0d7dq1Q2hoKGJiYgAA3bp1Q7NmzfDXX39ZfJysrCzUq1cPH3zwAV577bXqe0JEJDkO0xBRtSktLcW2bdtw//33G4IIADRp0gTjxo3D7t27kZOTAwDw8/PDsWPHcOrUKYv35enpCXd3d0RHR+PGjRuStJ+IpMEwQkTV5urVq7h58ybatWtndl2HDh2g1WqRlpYGAHj77beRlZWFtm3bokuXLpg+fTqOHDliOF6j0eC9997D5s2bERAQgAEDBuD9999HRkaGZM+HiKoHwwgR1QgDBgzAmTNnsHz5cnTu3BnffPMNunfvjm+++cZwzNSpU5GSkoKFCxfCw8MDs2bNQocOHZCQkCBjy4moshhGiKjaNGzYEF5eXkhOTja77uTJk1AqlQgKCjJc5u/vj/Hjx2PVqlVIS0tD165dMXfuXKPbtWrVCq+++iq2bduGpKQkFBUVYfHixdX9VIioGjGMEFG1UalUGDZsGNavX2+0/DYzMxM///wz+vXrBx8fHwDA9evXjW5bt25dtG7dGoWFhQCAmzdvoqCgwOiYVq1awdvb23AMEdVOXNpLRFVi+fLl2LJli9nlc+fORVRUFPr164fJkydDrVbjyy+/RGFhId5//33DcR07dsTAgQPRo0cP+Pv7Iy4uDr///jumTJkCAEhJScGQIUPw0EMPoWPHjlCr1Vi7di0yMzPxyCOPSPY8iajqcWkvEVWKfmmvNWlpabh69SoiIyOxZ88eaLVa9O7dGwsWLEBYWJjhuAULFmDDhg1ISUlBYWEhWrRogSeeeALTp0+Hm5sbrl+/jjlz5mDHjh1IS0uDWq1G+/bt8eqrr+LBBx+U4qkSUTVhGCEiIiJZcc4IERERyYphhIiIiGTFMEJERESyYhghIiIiWTGMEBERkawYRoiIiEhWtaLomVarxeXLl+Ht7Q2FQiF3c4iIiMgBgiAgNzcXgYGBUCqt93/UijBy+fJlo/0riIiIqPZIS0tDs2bNrF5fK8KIt7c3gLIno9/HgoiIiGq2nJwcBAUFGc7j1tSKMKIfmvHx8WEYISIiqmXsTbHgBFYiIiKSFcMIERERyYphhIiIiGTFMEJERESyYhghIiIiWTGMEBERkawYRoiIiEhWDCNEREQkK4YRIiIikhXDCBEREcmKYYSIiIhkxTBCREREsnL5MLIlKR1bkjLkbgYREZHLqhW79laX3IJiTPrxEADgxNsj4OmukrlFRERErsele0ZuFZUavi4sKbVxJBEREVUXlw4jUJR/KQjyNYOIiMiVuXQYUYjTCBEREcnCpcOIGDtGiIiI5OHSYSQlM9fwtcBxGiIiIlm4dBiZ8vMhw9eMIkRERPJw6TCSW1AidxOIiIhcnkuHEQVX0xAREcnOtcOIaDWNwIEaIiIiWbh0GCEiIiL5uXYYEZcZYccIERGRLFw7jIgwixAREcnDtcMIEwgREZHsXDqMFJVqDV9zNQ0REZE8XDqMiHE1DRERkTwYRnS0zCJERESyYBjR4d40RERE8mAY0blZVCp3E4iIiFwSw4jOpztOyd0EIiIil+TSYeS1YW0NX5+9mi9jS4iIiFyXS4eROxrWNXyt5ZwRIiIiWbh0GDGqBs8sQkREJAuXDiNi7BkhIiKSB8OITinDCBERkSwYRnSYRYiIiOTBMKLDYRoiIiJ5uHQYUYhmsDKMEBERycPpMBIbG4uIiAgEBgZCoVBg3bp1dm9TWFiIN998Ey1atIBGo0FwcDCWL19ekfZWG63W/jFERERU9dTO3iA/Px8hISGYMGEC/vOf/zh0m4ceegiZmZn49ttv0bp1a6Snp0NbI87+5V0j3JuGiIhIHk6HkZEjR2LkyJEOH79lyxbExMTg7Nmz8Pf3BwAEBwc7+7DVjrv2EhERyaPa54xs2LABPXv2xPvvv4+mTZuibdu2eO2113Dr1i2rtyksLEROTo7Rv+rGpb1ERETycLpnxFlnz57F7t274eHhgbVr1+LatWuYPHkyrl+/jhUrVli8zcKFCzFv3rzqbprRBFYO0xAREcmj2ntGtFotFAoFfvrpJ/Tq1Qv33HMPPvzwQ3z33XdWe0ciIyORnZ1t+JeWllbdzeQwDRERkUyqvWekSZMmaNq0KXx9fQ2XdejQAYIg4OLFi2jTpo3ZbTQaDTQaTXU3zQiX9hIREcmj2ntG+vbti8uXLyMvL89wWUpKCpRKJZo1a1bdD++wrJvFcjeBiIjIJTkdRvLy8pCYmIjExEQAwLlz55CYmIjU1FQAZUMsTz75pOH4cePGoX79+hg/fjyOHz+O2NhYTJ8+HRMmTICnp2fVPIsqkpFdIHcTiIiIXI7TYSQuLg6hoaEIDQ0FAEybNg2hoaGYPXs2ACA9Pd0QTACgbt26iIqKQlZWFnr27InHHnsMERER+PTTT6voKVScwuT77FvsHSEiIpKa03NGBg4caHPlycqVK80ua9++PaKiopx9KMkpTdMJERERVTsX35tGYfK9TA0hIiJyYS4dRoiIiEh+DCMiXN1LREQkPYYRERY+IyIikh7DiIgAphEiIiKpuXQYad/Y2+h7rVamhhAREbkwlw4jQf5eRt+zJDwREZH0XDqMmFqx57zcTSAiInI5DCMifxy6KHcTiIiIXA7DCBEREcmKYYSIiIhkxTBCREREsmIYISIiIlkxjBAREZGsGEZE2gbUlbsJRERELsflw4i3Rm34enS3pjK2hIiIyDW5fBh5flArw9ef7TwlY0uIiIhck8uHkUkDysNIQTE3pyEiIpKay4cRpVIhdxOIiIhcmsuHESIiIpIXwwgRERHJimGEiIiIZMUwQkRERLJiGCEiIiJZMYwQERGRrBhGTNzIL5K7CURERC6FYcSEVhDkbgIREZFLYRghIiIiWTGMmGC/CBERkbQYRkxwlIaIiEhaDCMAPnigq9xNICIiclkMIwAa+XgYvhY4UENERCQphhEARvv2MosQERFJimEEzB9ERERyYhgxwWBCREQkLYYRE1xNQ0REJC2GEROswEpERCQthhEAgiiAMIoQERFJi2HEhFbLOEJERCQlhhETHKUhIiKSFsOICRY9IyIikhbDCIzniXCUhoiISFoMIya4moaIiEhaDCMmmEWIiIikxTBiQmAaISIikhTDCGA0aYRzRoiIiKTFMGKCq2mIiIikxTBiQquVuwVERESuhWHEBFfTEBERSYthBByaISIikhPDCIC+rRsYvmbPCBERkbQYRgBo1CoE+noA4GoaIiIiqTGM6CiVCgCsM0JERCQ1hhEdpaIsjLBnhIiISFpOh5HY2FhEREQgMDAQCoUC69atc/i2e/bsgVqtRrdu3Zx92GqnyyLsGSEiIpKY02EkPz8fISEhWLJkiVO3y8rKwpNPPokhQ4Y4+5CS0PeMMIoQERFJS+3sDUaOHImRI0c6/UCTJk3CuHHjoFKpnOpNkYq+Z6SU4zRERESSkmTOyIoVK3D27FnMmTPHoeMLCwuRk5Nj9K+6lc8ZYRghIiKSUrWHkVOnTmHmzJn48ccfoVY71hGzcOFC+Pr6Gv4FBQVVcysBlX6YhlmEiIhIUtUaRkpLSzFu3DjMmzcPbdu2dfh2kZGRyM7ONvxLS0urxlaW0Q/TsGeEiIhIWk7PGXFGbm4u4uLikJCQgClTpgAAtFotBEGAWq3Gtm3bMHjwYLPbaTQaaDSa6myaGZWuzgjnjBAREUmrWsOIj48Pjh49anTZ0qVLsXPnTvz+++9o2bJldT68U5QcpiEiIpKF02EkLy8Pp0+fNnx/7tw5JCYmwt/fH82bN0dkZCQuXbqE77//HkqlEp07dza6faNGjeDh4WF2udyUXE1DREQkC6fDSFxcHAYNGmT4ftq0aQCAp556CitXrkR6ejpSU1OrroUS0ZeD55wRIiIiaSmEWlByNCcnB76+vsjOzoaPj0+1PMbYL/Yi/sINLHu8B0Z0blwtj0FERORKHD1/c28anVtFpQCA3IJimVtCRETkWhhGdI6nlxVWm/77EZlbQkRE5FoYRoiIiEhWDCNEREQkK4YRIiIikhXDCBEREcmKYYSIiIhkxTBCREREsmIYISIiIlkxjBAREZGsGEaIiIhIVgwjREREJCuGESIiIpIVw4jOwz2DAADDOwXI3BIiIiLXwjCiE9ygDgCgrsZN5pYQERG5FoYRHaWi7H8BgrwNISIicjEMIzpKRVkaEZhFiIiIJMUwoqPLItAyjRAREUmKYURHwZ4RIiIiWTCM6CjZM0JERCQLhhEdXRZhzwgREZHEGEZ0lLquEa6mISIikhbDiI5+zohWK3NDiIiIXAzDiI5+mIZzRoiIiKTFMKJjqDMiczuIiIhcDcOIjr7OiMCeESIiIkkxjOgYysEzixAREUmKYUTHMIGVaYSIiEhSDCM6+gms6dkFsraDiIjI1TCM6FzJLQQAnMzIlbklREREroVhROfM1Ty5m0BEROSSGEZ03FV8KYiIiOTAM7COu7r8pSjVchIrERGRVBhGdMQ9I0UlrAlPREQkFYYRHZW+0Ai4vJeIiEhKDCN65VmEJeGJiIgkxDCiowB7RoiIiOTAMKKjFPeMMIsQERFJhmFER79rLwCO0xAREUmIYURHnEU4TENERCQdhhEdhSiNMIoQERFJh2FEx3jOCOMIERGRVBhGLGABViIiIukwjFggcKCGiIhIMgwjOuI6IxylISIikg7DiAUMI0RERNJhGLGAwzRERETSYRixgBNYiYiIpMMwotPEz8PwtZZphIiISDIMIzpjuzczfL3paLqMLSEiInItDCM6KlHVs7QbN2VsCRERkWthGLFAreTLQkREJBWedS3w9lDL3QQiIiKXwTBiQa+W/nI3gYiIyGUwjIh0aOIDgEXPiIiIpOR0GImNjUVERAQCAwOhUCiwbt06m8evWbMGQ4cORcOGDeHj44OwsDBs3bq1ou2tVvo5rMwiRERE0nE6jOTn5yMkJARLlixx6PjY2FgMHToUmzZtQnx8PAYNGoSIiAgkJCQ43djqptCFES27RoiIiCTj9EzNkSNHYuTIkQ4f//HHHxt9/+6772L9+vX4888/ERoaavE2hYWFKCwsNHyfk5PjbDMrRKlg1wgREZHUJJ8zotVqkZubC39/65NEFy5cCF9fX8O/oKAgSdqmrzTCnhEiIiLpSB5G/ve//yEvLw8PPfSQ1WMiIyORnZ1t+JeWliZJ2xS6nhFmESIiIulIWlDj559/xrx587B+/Xo0atTI6nEajQYajUbClpXhnBEiIiLpSRZGVq9ejYkTJ+K3335DeHi4VA/rFP2cEUYRIiIi6UgyTLNq1SqMHz8eq1atwqhRo6R4yArRzxkR2DNCREQkGad7RvLy8nD69GnD9+fOnUNiYiL8/f3RvHlzREZG4tKlS/j+++8BlA3NPPXUU/jkk0/Qu3dvZGRkAAA8PT3h6+tbRU+jaig5Z4SIiEhyTveMxMXFITQ01LAsd9q0aQgNDcXs2bMBAOnp6UhNTTUc/9VXX6GkpAQvvPACmjRpYvj38ssvV9FTqEKGOSPyNoOIiMiVON0zMnDgQJvDGCtXrjT6Pjo62tmHkE15BVamESIiIqlwbxoRha5rhD0jRERE0mEYEVHqXg1OYCUiIpIOw4gIJ7ASERFJj2HEAhY9IyIikg7DiAh7RoiIiKTHMCLCcvBERETSYxgRYTl4IiIi6TGMiLAcPBERkfQYRkQUnDNCREQkOYYRkRPpOQCA9YmXZW4JERGR62AYEbmUdQsAsO/sdZlbQkRE5DoYRoiIiEhWDCNEREQkK4YRIiIikhXDCBEREcmKYYSIiIhkxTBCREREsmIYEXluwB0AgM5NfWRuCRERketgGBHp0KQshPh6usncEiIiItfBMCKiVJaVg9dqZW4IERGRC2EYEVHp9qYp5eY0REREkmEYEdF1jECrZRghIiKSCsOIiGGYhj0jREREkmEYESkfppG5IURERC6EYUREZZjAyjRCREQkFYYREYV+zgiHaYiIiCTDMCKi7xkpZc8IERGRZBhGRPRzRtgzQkREJB2GEREle0aIiIgkxzAiomYYISIikhzDiIhaVfZyFHNtLxERkWQYRkT0PSMl3JyGiIhIMgwjIm7sGSEiIpIcw4iIWlXWM1Jcwp4RIiIiqTCMiKi5Nw0REZHkGEZElIa9aRhGiIiIpMIwIqKvM1JQrIXAQEJERCQJhhERXRYBAEQnX5WvIURERC6EYUREXw4eAJIuZcvYEiIiItfBMCKiEIUR0ZdERERUjRhGRFRKJhAiIiKpMYyIiLMI568SERFJg2FERClKI8wiRERE0mAYEVGKJoqwZ4SIiEgaDCMiGnX5y9HA213GlhAREbkOhhERN5US3h5qAEA9L4YRIiIiKTCMmOgU6AMAKNVynIaIiEgKDCMm9PNGuFkeERGRNBhGTKi4cy8REZGkGEZMGHbu1crcECIiIhfBMGLC0DPCOSNERESSYBgxoa97xmEaIiIiaTCMmDAM0zCMEBERSYJhxASHaYiIiKTldBiJjY1FREQEAgMDoVAosG7dOru3iY6ORvfu3aHRaNC6dWusXLmyAk2VhtKwmkbmhhAREbkIp8NIfn4+QkJCsGTJEoeOP3fuHEaNGoVBgwYhMTERU6dOxcSJE7F161anGyuF8tU0TCNERERSUDt7g5EjR2LkyJEOH79s2TK0bNkSixcvBgB06NABu3fvxkcffYThw4c7+/DVTsUJrERERJKq9jkj+/btQ3h4uNFlw4cPx759+6zeprCwEDk5OUb/pKJSlr0kJewZISIikkS1h5GMjAwEBAQYXRYQEICcnBzcunXL4m0WLlwIX19fw7+goKDqbqaBm4rDNERERFKqkatpIiMjkZ2dbfiXlpYm2WPrV9OUlDKMEBERScHpOSPOaty4MTIzM40uy8zMhI+PDzw9PS3eRqPRQKPRVHfTLFIr9T0jrAdPREQkhWrvGQkLC8OOHTuMLouKikJYWFh1P3SF6OeMFLFnhIiISBJOh5G8vDwkJiYiMTERQNnS3cTERKSmpgIoG2J58sknDcdPmjQJZ8+exYwZM3Dy5EksXboUv/76K1555ZWqeQZVTK2bM7Is5gwKiktlbg0REdHtz+kwEhcXh9DQUISGhgIApk2bhtDQUMyePRsAkJ6ebggmANCyZUts3LgRUVFRCAkJweLFi/HNN9/UyGW9QPkwDQA8veKAjC0hIiJyDU7PGRk4cCAEGzU4LFVXHThwIBISEpx9KFmIw8g/Z/+VsSVERESuoUauppGTfs4IERERSYNnXhMCOHGViIhISgwjJg6c49AMERGRlBhGTLDyKhERkbQYRoiIiEhWDCMmTPtFcguKZWkHERGRq2AYMWWSRrrM3YbU6zflaQsREZELYBgxYWk1zZqEizK0hIiIyDUwjBAREZGsGEZMeLipzC5TQGHhSCIiIqoKDCMmZgxvb3aZglmEiIio2jCMmGji5yF3E4iIiFwKw4gJS50gV3MLJW8HERGRq2AYccAP/1yQuwlERES3LYYRIiIikhXDiAkFZ6sSERFJimHEBKMIERGRtBhGTLir+ZIQERFJiWdeE3U0armbQERE5FIYRoiIiEhWDCNEREQkK4YRC8Z2byZ3E4iIiFwGw4gFLRt4mV1WXKqVoSVERES3P4YRCx7qGWR22ZcxZ/D+lpOIv3BDhhYRERHdvrh0xAI3lXlG+9+2FADA0ugzOL9olNRNIiIium2xZ4SIiIhkxTBigZdGJXcTiIiIXAbDiAUaNcMIERGRVBhGKqCguFTuJhAREd02GEYq4MbNIrmbQEREdNtgGKkApYJ7+xIREVUVhpEKEEeRq7mFKNUKsrUFAK7nFbIoGxER1VoMIxWhSyMJqTdw54LteGr5Admacu5aPnrM346Iz3bL1gYiIqLKYBixQqO2/tLMWpeEIxez8OM/qQCA3aevSdUsM38dvgwAOJmRK1sbiIiIKoNhxIqfn7nL6nVbj2Xivs/3QCkar1l9IFWCVhEREd1+GEas6NGiHnoF+9s8RjyRdeaao7iSW1DdzSIiIrrtMIzYoHGz/fIolcaravILna8/8t3e8xj16d+4nlfo9G0BQN6ps0RERJXHMGKDvSW8f5+6avS9owt+kzNysXhbMnILijFnwzEcu5yDj7efsnisIDBuEBHR7Y279tpgr5zIxRu37N5HUYkWB879i+Pp2ZjQtyXUKiWGfxwLALiWV1487ZaFqq4vrkrA+Wv5WDu5D9QWdhK252ZRCQQBqKPh20xERDUXz1I2OFvc7OilbLSo74WYlKvo0MQHAT4eGPZRDM5fvwmgbOLrKtHE2KRL2Tbv70/dSpmD528grFV9i8dY6zjRagV0nL0VAJAyfyTcbawOcmWlWgGTf4pHxya+eDm8jdzNISJySTxD2aB0stDqi6sSsDkpA0+vOIi+i3YCgCGIAED8hRtYGn3a8L046wgCcKuo1OK+NxUZqrkpup9rFZyPUlPtOnkFcef/rZL7ik25iq3HMvHR9pQquT8iInIee0ZsUFSg7Pvknw4BAEqsVGXdlXzV4uVFpVp0mL0FXu4qHJs33OixK1vg9XaadXI56xbGrzwIADi/aFSl7+9mETc9JCKSG3tGbKjsDjS/x1+0eZ/ir9P+LetBuVlUiiKT0u6CjThh7brbdfeczJyqXT5t67UlIiJpMIzYUNkN8V777bDZZUZ3KfpGPBTTa8EO7Dtz3fB9bdx3prhUi7jz/1Z526s6Osi8rVClrE+8hI1H0uVuBhFRpTGM2DBlcOsqv09xFjmclmX4WnxSzL5VjEe//sfw/YSVcRbv62ZRCW7kF1m8Tm6z1iXhgWX7MP+v43I3xabaunQ6+2YxXl6diBd+PmRxnhERUW3CMGJD56a+WDH+ziq9z0OpWRYv19o5Kc7dcAwXb9w0uqzHO9vx3b4Ldh9TjhPu6oNpAOBQ+5xR0b6q63mFyL5ZXKVtMSUIgmTBIK+oxPC1tflJVamwpNTs54+IqKowjNjhoVZJ8jildk4oK/eex4SVB1FQXIqnVxzA17FnLdYm0RPf2+t/HIEgCHj99yN4aVUCnvk+zu6yYkfFplzF4P9F42AVrW6pDreKStFj/naEvL3NLJhVZU6bsioB7WdtuS1P2qM/34N+7+3CodQbcjeFiG5DDCN2SNWr4MiuuymZeXhj7VFEJ1/Fgk0nbB4rbvee09exeFsKfolLw4bDlxF1PBP/Wbq30m0uKtHiyeUHcPZaPh5ctg/aGjoB43J2eXE60yZW5QRW/fyNn/bffpsm6n8+1yVckrklRHQ7Yhixo1NTX7mbYGTNIesng5tFJVi8LRnBMzdig65gmt7nu04bfW+6YseaOeuTMPmneIuhrN97O42+/37feYfuszIqEh1s5UltNcwNdrY+TUWIH6K2znshx2m1At5YexSruDs43aYYRuzw9XSDr6eb3M1wSMfZW/HZzrLQ8ebaJLvHn76SZ/eY7/ZdwKajGUjOLO+5iU25iqXRp3El17iYmiM9AkmXsg3LmPMKSxC55gh2n7qGf/OLEJ18xanelYqchM2GaZy+B/squwrLEeJ219AOKapCu5Kv4Of9qYhcc1TuphBVC4YRB/jXcZe7CdUi/MMYm9eLT9ziHoQnlx/A+1uSzY+383iXsm7h3s92o//7uwAAn+04hVUH0vD4t/vR/Z0oPL3ioGHiqyMEAdhz+hrOX8t3+DYr9543el7V0avw7e5z0g5ZSflQVfBYt4o4GdZZ2beqd/I1kdwYRlxc8MyNWG2l61d8PnX0w/4X0WcQPHOjxetSMo3nxaRZOCFtTnK8bkbS5Ww89s1+DPxftMO3mb/xBHacuIIS3TBVdZzHbxaVYm3CJeQWFOOT7adw+krZ8z52ORvvbTmJvMISO/dgnSAImLo6ATP/OGK4zHQlliAIWJtwESfScyr8OLZcyrqFVQdSK7xyaMAHu9DvvV04lWl/nhQRuQaWg3dAPS83nJO7EdVo5pqjeKhnEBLSstChiTe83NVIzsjFpB/jDcfohx6ijmdavR9BEPDelpMWr4tOvoIYK6XwxYpKHJ/EcdTOiqCC4lL8FpeGZvW8jC6f+H0cGnpr8E/kEORXIhjYkpyZi7gLN7DqQCo+2p6C84tGYdSnuwEANwtLMG905wrdb0ZOAdYlGs8HMg0jO09ewSu/lBXc++Kx7rjrjvqoZ6V3TxAE3Cwqtbmz869x5b1VAgQM/ygWeYUluHTjFl4b3s7p53BVN7y38+QVtAnwdvr2RHT7Yc+IAz58qBvuDK4ndzOq1U8HUjH2i714/Jv9AIAJKw/inGj4Qz8p85nvLRdgA6z3Mmw4fBlPrziIlXvPGy67cD0fxaXmt0jPdrzc+97T142+z7pZZBRmlu46jVnrjxn2shG7mluIVm9swrw/HS/KdinrFr75+yzOXs1DYYntXoFSrYAEK8tgj1eix8LSEnDTi8Qh7fmfDhkV0DM1d8MxdJqz1erGgxdv3MSM348YXabv2fn79DVHm22Rq0x1EQSBk4xdXFGJFluPZSDrZs0sUlkTVCiMLFmyBMHBwfDw8EDv3r1x4MABm8d//PHHaNeuHTw9PREUFIRXXnkFBQVVu8dIdQpuUAe/TepjdFnf1vVlak31mLWubMLrodQsfLD1JC5l3TK6Pu3GTYz4ONbmfZy9annuxkurEswuu/uDaIu9LPpfVkEQ8OvBNGw9loFJP8Rj89F0xF+4gQPnyk+aG4+WD+nsO3Md3d6OwrCPyufBfLrTeAWRPXvPXMOsdUm4WWS5t2TMkj2Yv/EEBi+OwfCPYm2eYEq1giQTWQH7815sLRvXF6V7YNk+rE0w30spy6RQ3MUb5T8XVfXsopOv4KFl+4zC7+1CEAQ88e0BjP1ib41d+l6VElJv8IRrwac7TuG5H+Lx8JfWPxi4OqfDyC+//IJp06Zhzpw5OHToEEJCQjB8+HBcuXLF4vE///wzZs6ciTlz5uDEiRP49ttv8csvv+CNN96odOPl9MOE3nI3odos2XXG7LIJK+McqoVSWTkFJThyMQstIzdhxh9H8NwP8dhyLAPP/3QIY7/Yi0WbLQ8D6T/9n79e8YmR477ejx/+uYAlu05DEAR8v+88ElJvIPtWMUq1gtHqofPXb6Lfe7twI7/I4iffUq0Apei3a8bvxvsU7Uq+gn1nrmPKz4ewfLdjg4DbjmVgjIX6MOJhFKDik0z1Qzu2RDsw1Oasp1ccxIHz/1oMrWJzNxzD2C/2Wt3vSBAEfBSVgu02hhKldqu4FLtPX8Oh1CyzgO8MiXKtwaWsW3hq+QH8fcrx9zsm5SrGLN2Luz+Irr6GSaRUK2BZzJkqK/KnL7WQbGWelL2e1qomCAJOZeYa5s7VBE7PGfnwww/xzDPPYPz48QCAZcuWYePGjVi+fDlmzpxpdvzevXvRt29fjBs3DgAQHByMRx99FPv3769k0+WllKKYhIuau+GYrI+/LuGyxUBm6lLWLYS+E4XQ5n7w9zKek7HjRCYaeGsM3/8aV97rcCW3EONXlA8d/XUkHRP6tbT7eM/+EG/x8v9tS8GUwW0M31vqKREEAYoKnNGkPAlezyu0eLlWK+D9rcmGYb5l0Wdw8cYtvDK0LRr7ehiO23HiCj7ZcQoAcH7RqGpvryOqanSmqkd5ruYWYlfyFUR0DYSnu3mV6Rm/H8ae09cRk3LV4dcy6ngGgMqt/Mm+VVwtpRSOXMxCgI8HAnw87B8M4I/4i4YPPlXxs2Tr9+j0lTyEfxiDcb2b490xXSr9WI74bu95zP3zOCJCAvHZo6GSPKY9TvWMFBUVIT4+HuHh4eV3oFQiPDwc+/bts3ibPn36ID4+3jCUc/bsWWzatAn33HOP1ccpLCxETk6O0b+aoH1jTraTgrX9e6Ti7CfYhNQs7Dhp3DN4ObvA6sn/So75SffLmDMI/zAG16yckJ1x+qp5/ZjQd6Iq1GNga6hJf9WGw5erpODdZSvzhTYlpWNZTHk4XBxVVk34xVWHjI5Lzym//fHLjv/NMN8ioOpq3RjXgxFw/5I9eNFOD5Alu+3Mz/l4ewrGLN2DW0WOfcJ+cNlezPj9CN61Usk5w4m5W3riUajTV/LQZ+EO/PjPBVzPK3Ro5dXGI+kImbcNH2y13PtZUSfSc3Df53vQ+90dDt/GdOWfXkFxKb6MOeP0SjBbmX5pdNlw8s8SVm5eEl32+/SnSXFMOTkVRq5du4bS0lIEBAQYXR4QEICMjAyLtxk3bhzefvtt9OvXD25ubmjVqhUGDhxoc5hm4cKF8PX1NfwLCgpyppnV5vfn+yByZHvETB8od1PIjpowYdDaHyBLewot3HwSp6/k4TPdJ3sAuJJbgBPpORi9ZI9D3eVFJVokXcrGpqPmv4tZN4sx0cbkY1P618/WJzr9VS+tSsDs9cecqvfiDGsnxpPp1k8I93z6N67mFtr9OfhsxymELdyJdN2WAasOpOLOBTtw7LLllVpXcwsNK7C+jDmD3u/uQKrJ0ODZq3l4f8tJsx217/4gGolpWRU6AZhWXr5ZVIL/rjyI3+PLetw+3n4KCalZ+C3esTo9+uHMbcct/90+Y2X+ly3il3rWuiRczi7AW+uS0GP+dvR7b5fF2xSXavHAF3sxd8MxvPBzWbh0pFfSGbb2zTp2ORsvrkrAhevGz9faz/3SXaexcPNJDP3I9vw5U7Z6JRVVNvuqdqv21TTR0dF49913sXTpUhw6dAhr1qzBxo0b8c4771i9TWRkJLKzsw3/0tIcL4RVnepq1Hju7lZoUb+O3E0hGw6nZWHAB5b/+EmpIiN53+27gOybxfg1Lg29FuzAyE/+xuG0LDzxre1J4gAw+ad43PvZboce50Z+EdYnWt5aYOHmE2j31ha8u+mEYRmuJYdSszB1dfmnfEe65x0tdnYiPQfd3t6Gab8kWj3GdEmz6ct954LteHqF+UqqW0WlKCwpRer1m1gclYKMnAJ8sr0sBEauOYpreYV49Vfz+TPX8gpx54Lt6DE/CkBZgLySW2jWuxDx2W4sjT6DGboNKitqS1K61VA08bs47Dh5Ba/9ZtzO9zafrPREWdOVVfo6OfaVP26JyT4L1nr8YlOuIu7CDaOVdkDZz1Jlhnsu3riJz3eeQtbNIptDXBGf7cafhy9j4nfGQd1aeEhIy6pQeyzdW9z5fzFkcTT2nqncqrTbhVNzRho0aACVSoXMTOPu3szMTDRu3NjibWbNmoUnnngCEydOBAB06dIF+fn5ePbZZ/Hmm29CqTTPQxqNBhqNxuzymqZrM18cuVg1u99S1Rm9ZI/cTQAAqCo4ryjk7W1O32b2+iRsP2F5ErlY27c246eJvTHvz2NIumQ+lPH3qav4MuYsAOCr2LP41s7kWtOaJ7Z88/dZzN9oe4NHvUe++gfZt4qxJuES1ljZnE9/jtFqBfx55LJhmwGxmJSr+HTHKfwWn4Y1z/dF2o2bFjeJdCQzJOiGDwuKjU+0pqEoXzdUEn/hRoWXLyemZWHSj2U9BZbmLOw9c93sMv1jb07KwKiuTSxen5lTgFQLr5PpY4uFfxiLtZP7ILS57fIG4pfB0U/71nJTyLyy34HTC0ZCrXL+M/PYL/YiM6cQRy5mo08r6ysf9Y9/xmRoU4q+ioe+3FdlWzlotYLZPMaC4lJ4uEmz63xVcOpddnd3R48ePbBjR/nYm1arxY4dOxAWFmbxNjdv3jQLHCpV2QtUE7rSK2PN831w8M3y+TPPDbjD8HVTP085mkQ1iJTdr9/rlujaU1SixYPL9lkMIgDMemAs1TWxxt6RpkHE0q//H/EX8dKqBIc+FetvvybhEl5enYgvY89aPO7DqBSk/XsLS3adtrpbtSMTdcV/r8QVcIGyeTN9Fu7AUdGHk3/zbX8q17uUdQtvrj2K01fysDT6NFYdSHVo3yhrdp60Hkp7v7sDDy6zPL9Pz1KbLa3iAsqGWZ5cfgDPfB9nXDfIwR99e3k932QOzMo95/B17Fmcysy1ef7I1M3L2nfmutEJ39ptzHpCRN9W9jy1eFsyzoqGMBduLvs9qKogkpyRi25vb8PXop//j6JS0H7WFuytZC0gKTkdOadNm4avv/4a3333HU6cOIHnn38e+fn5htU1Tz75JCIjIw3HR0RE4IsvvsDq1atx7tw5REVFYdasWYiIiDCEktpKrVKiobcGj/YKQtuAuhjWqXwuzfopfWVsGZH09OP/CzaaF5KztkTSNOy8+tthsx2nrdH3SBw8Z31OgK3HEjM9F1mauPvx9vL5PKZ7KL20KgGXswsM8x703nagqN7zP8bjp/2pCP8wBu9vSUbkmqM2z+V/HbH9+vxx6CI+ikrBfZ/vtlozR89SYL6W7/gk6k1H0xGbchVRxzPxx6HyFWO22i8+udtblSi+Or+wBHP/PI4Fm05g6EexWL7nvP0GKoxD8riv9yP+wg2ntkoY/nEsXtcV/nM2l+QWFBs2L9XT9zw669jlbLy8OsFsjtJb644ip6AEC0TDhfpVZXNkXpnoDKeX9j788MO4evUqZs+ejYyMDHTr1g1btmwxTGpNTU016gl56623oFAo8NZbb+HSpUto2LAhIiIisGDBgqp7FjJb+J+uAGD0qaj+bbq5HjnugI2Jc7ej7ScyEXfhBuIu3ECPFvUwonP5UIGlHonreYXoqZt/URH684Kjy48FG303qw4YhwtxZ25xqRaxKVcdqpxrWrdBfII2pdUK+Gb3WYtDvZuTyieWmn4yn/Kz8WqcJAvbIuhPRqsPpDm0bFzMmZOltX2W9lsIiNfyCrH1WAYWb0vB9xN6oXNTX6jsvHn6HosjF7OMwiAAvPPXcfzXznPLLSgxmkOz7+x1jP2i7GdRPPxl2gpxSEvJzENKZh7ee6ArbogKuhWWlOJKTiF8PNzg61W+HPn8tXwE+XtBpVRYrDLtqOt5hahft3y6wujP96BEK+Bkei62vjKgwvdbU1Vob5opU6ZgypQpFq+Ljo42fgC1GnPmzMGcOXMq8lC1irdH+ctZkZoORLVZbEp5l/CkHw/hybAWNoePvnGw2Js1Wq2Aa3mFDm88+OM/tpdOigOJfl7Iu5tOYOWe8yhysDiUteXJphJSb2DR5pMWT9pAWbDTS8m0PWRzwUahv7f/Oo7H72qBIxezEBLkZ/W489fy8fBX+xA5soPthptwpqeg5/zthq9f/fUwhnRohKXRtlfO6IPYfZ/bnwd2+koeGtbVGAUDAEY9BmLxF6x/WLD053vz0XQcEy0Zz8guMBR42zClL7o288OqA6mIXHMU/wltig8f7ubUMKfY4m3J+GznaTzaK8jwYbdEd1+mhdNMe7esDSslpN6ASqlA12Z+Rrf4ZPspvBzexuJtpMS9aapQcIM6iBzZHh880LXC92Ft4hlRTWfa9e3oPJaKKtEK6Dl/O/464vhOz47Sz9n4Kvas3SCyrQL1Wz7Ymmw1iJiKsLNCyl6V0LZvbcYDy/Zh3p/mXfYZOQXYcPgyBv4vGpk5hZhqY/WSJfEXKlahtFQQ7AYRwPH9i5IuZSP8wxinJn+P/aJ87kyJVkDwzI34Q7dU2tJHyed/Mh6CE/do3ff5HmTdLMKnuh4p/aRra8HAWsXVHScycfZqnmFoZ9WBNJvl9U9fyUW+aChu27EM3LnAvJ5KfmEJxizdi/s+32O2GelH21Os3r+UGEaq2HN3t8KDPW3XRZkxwvpOpzNHtK/qJhFRBVTnBHtnarLYC0P2VjzpWesZsleGX+/f/CJsPJJuWKYrCALWWlnpZI/pCiRrMrMLLK6SMjXOxmaQznj1t8MQBMHQC2GL6byiExbq3hRa2YW83VtbLF7+3+/iMHhxjNFl1kLrwfP/IvzDWKPemmd/iDdaRn3qSh7m/3Ucd4kKvkldet5RFRqmocoJ7xCA97ckW7yOZeaJaoZPdzi30aIzHB3OqUm6v1M2vyfI3xOLH+yGVQcqXjHU2qaapoZ+FIvhnQLsHpdT4NhQnSOOXMzGV1ZWZomZDuW4qRRGw1ap129WSb2j536Ix9LHuhtdtiUp3W5VXj3T4VBLoakmYBipYZhFiGqGmtJ9XdOk/XsLD31pe3lwVdp6zPowmFYrICPHONg5OofIGkfrFJn+qT57Ld+oLVVZeHGyyRCRvgZNRYz7+h/4myywOHs1D3c0rFvh+6wKHKapRi8Nbm3xcrUocfw2yXJ9FgA48MYQJM8fgUHtGlZ524iIarsBH+xCn0U7jS5btNmxwnpVbcbvR+wfVANYGoIyHRqSA8NINZoyuA2eG3AHPnmkG4Z1DICflxuWjOuOlg3qICIkEE/c1QJ3Bvvjsd7Ny44f1NqoEI5GrYJGrUKnQF+ZngERUc118Yb5ppYHz1VsUq2zTCe01iZXLGzzMHhxdIU2SKwqHKapRu5qJSLvKVsqN7pbU6PrxNs2vz26Mx7r3QLtG3vjcnb5L5dKVdaDwlXCRESOMV36So45ezUf+89dNztXSYU9IzWASqlAx0AfKJXGE6CqYv7IqC7GS4VfHdoWs+/tWPk7JiKi24rawl5xUmEYqWHES970xWye6hNc4YquHz4cgg5NfAzfvzikDXtaiIjITEU396wKDCM1TBNfT6iUCtRxV0GjLnt7GtTV4OCb4Tj4Zjie7hOMrVPLSwG/Et7W6n31b9MAGrXKrF5CPS/pS9Xf08Xyrs5ERFQzMIyQgbtaiaS5wxE/a6hRzRGlUoGG3hrMva8T2jX2Lr/c5Gdn00v9DV9bK0l/b9cmGNu9GcbpJs7aE+jrgZeGVK5csHgp2YE3huCP5/tU6v6IiKhqyVlagmGkBvJ0V8HDzbEdjU2P6xhYPiSj/7kyrXaoVimx+KEQPNv/DoceY8/MwZg21HoPjCPEY5GNfDzQo0W9St2f3D4fF2r/ICKiWuRWsXzVWRlGaqlXh7ZFSDNfm70b+o6RuhrLi6aCG9TB3IiO+OSRbtgXORgx0wdauR/zuCyeh+IIS1uyy8naa+KI98Z2QbsAb/sHEhHVImoO05CzXhzSBuun9EMdCydV/bbarw4t2wPngwdD0CnQB1+YlBQGgKf7tsTobk3RxNcTLerXsfmYddzLe2HuCwk0fD373o7wM9kps6by9lAjZf5IeLo71vNkSaCfp8O7uBIR1R4MI1SFZt3bESffGYEuzcqKpbVqWBcbX+qPkV0c3xF4fN9gAMDQjuX7QogHe57p39LwtVIB/O+BEADAzJGWN/ob3L4RAONAoze1mravttSWJr4ecFcr8dUTPSp8v/1aN0CBjN2ZRETVQc4ObIaR25Sjc05MbXqpP5Y93gNzIjrh6NxhRift13U7Cj8V1gJqlfGPTnjHAJx4ewQm3d0Kj9xZtmtx/zYNDNf3aumPzS/3x96ZQ8weUwEF9r8xBJPubmWzbS0b1EHs9EEY3S3Q4vWHZg01+t5Sr5F+uXRo84rPWVEoFGjq51Xh25tq5K3BL8/eZXRZ24C6CPL3rPR9D9GFQCIie+QcTGcF1tuASqlAqQNbXjuiY6CPYRKst4fx0MtTfYIxuH0jNKtnfJLUzynRD328c39nPNgzCCHNfJGSmQelsmyVkLV5JgIEBPh4YObI9lgWc8Zq23a9NhAAMHlga6xPvGx2venmTypF2Qqkq6LSx73v8Ld6/85o7OtRJfcDAG4qJXrfUd/osj9f7IfRn1vfsOverk0w975O6Dl/u837nj+mM7w2ncSfh81fL2do1Eqr26E7w12l5BAXUQ1lbQWmFNgzchv4bVIYQpr54o/nrW+6V1WC/L3s/sC6qZTo0aIe1ColOgb6oH1j5ya72tOusTeeHWB/JVBESBNsfLEfPno4BFGvDEDkyPZWh5GAst4be6YPb2f4euX4OxES5IeV4+80OuaNe6w/hiWWXk6NWoVPHrG8YuevF/vh83HdjQrhrRC14eGeQUb3Y1pnpiK8PdRY/0LfSt/PodlD0b5x1U/+ffwux5apE5F1cvaMMIzcBro3r4f1U/qhR4uq+dQvpzuDjYdPPN1UaO7vZbSXDwC8cU8HnF80Ch880BUAEGbSs9Ap0AfeHm5o5OOBMaHN0CbAG8/d3Qpe7uWdgW/q9g3S0xeZs+WFQeU7MQ9s1wjrX+iLge2Mh0KeHWB7uMmUp5UhtXYmJ+3t0+5GzPSB6Ny0bC6QOBSK/4g0r++Fp/sE4+k+wfCv446KZJGF/+li9P3yp+90ejzZ9L3Ut/OBHs2cb5AdoUGODbtp1EqH6+tQ1YsIsTzESjUD54xQrVaVP8BLH+uB50S9HjtfuxuxMwZZ/SP2QI9m2PRSf6ycYNw74UglwWcG3IHDs4dh78zB2P/GEPh62l4R5Kaqnt/Ulg3KVjGZDjOZat2ortUVT6a9VXPv64S593UCAKdXDo3vG4xHexmfsLs28zPMtzFlrWZMs3rm82oUiurpCg5t7odvnuxp9zitIOD1Ee3xdJ/gSj3evV0dnwxO5T55uJvcTSAbGEaoVjOdQ+Is/ckYABp6azBlcHnvg71P9QpF2SaDGrXxCdeRXg4A8PVyQ6CfJwJ8PPCGSU+JWK9gf/w+qXJVY00Lpf3w314Y1jEA88d0BlA2/AJUvAaKfnn1IJOemhmioaVRDpxE9b1MT9zVwuhy8R8qcVipo1HDw8389bb0d01hNdLYtu6Fvlj2eA+cefcei9crFAqEdwwwLGvXe+Oe9tj2Svn2CSqlAr6ebph7Xyd8N6GX0bGrTSYR2yIuLlhR4uG0290b97TH9ml3G1WVro1aNbRd/qC2M50nKCWGEaqwnyf2xhv3tDc7+Tnq1+fCMGNEO0R0Ne71qMwn5/cf6Irg+l5YNLar07cN9LMeqn6dFIaQID+H70v/yVt8kr7X5Hn2b9MQXz3ZE428PQyPvy+yrJfGWQoAu18fjF2vDTQ7UTby8cCyx3vgjXva41Mr81AAoLm/F758oodhObcA60nwXV2AAgCtVnA4YigU5hWBAeA/oba3LW/krcGIzo3t9ni9bLJMvG/rBmgb4I1lj/dAI28NvhtfHkDubtvQ6Ni77qiPd8cYD09ZYzosWBHv/qeLrCc3KasgPzugFVo3qgugds/vqV9XI3cTqlVPGStjM4xQhfVp3QDPDmhV4fDQq6U/Jg9sbfPTkrPTHR7qGYTo6YPQqmHdCrXJdIltRb1xTwf8PLE3EmcPw+SBrbDsccfqmjTx9TRakuxopVuFoqxHRdzLJDaic2M8O6CVzZN5kL8nhndqbHg/TTOD+G0Wv+daQcD7D1gIfxYeShCAEtHKr70zB+Od0Z0wf0xnh3uzTHm4KdFUFyR9PNzQyLv8hKGv/Duic2MceDPcbNWSqYHtGtq8/uOHu2HN5D4IbV4PJ98ZYXVbAEd6t1RKBd65v7Pd4yrK2s+C3qePWg+m9jgTZEznDs2/vwu8K1EBWU61u1/Hton9WnI1DZGY+NehKlaCOMPeycoacU0VoGwpc5/WDeDhpsKMEe0xonPFdi329azeP9pvjbI+NOXoK68VBIcnJgoQUCxaIhzo54knwoLh5a7GmzbaYu1v5FdP9EDi7GFwFwWZAJ/yZdeObkOg78GytkS+d0t/zLq3I+4PbYruuho1Hm4qs94uvV+fc2xlm7h9XzpQiO+7Cb1svmdiHzzQ1epk4anhbQwBriJUDr6uDepqsGSceeXnv17q59DtLRVJFDtrMmz3aK/yoa/t0wYgcfZQnF80ynCZfx13rHjaeH6ZMyqzjURNJ+eOvQDDCNVAnm4q1HFXQaVUGJ1YpGZvQqnYYAeLi+lPUlW9t03FZmIAE21slmiaA629HtZK3Ji2qU+r+vB0U6GplTlGT9zVAlum9sfJd0ZYb7DIswPuwLBOjc0K/ImHl4IbOFacLlg3Mdha/Zhfngszm49ii705JfoeOHEYGd7JdmD9bVIY7m7bEBP734F/IofY7VLvGeyPKaLVX2JTw8s2vnRkyCREV8lZzNEP0L9NCkMjC7/DLerXcWgScWs7vydKpQJbpw7AO6M7YcOUvhjdrXy4r3Ujb/h5Gf/Mzr+/MwbZ+F3tZOF9E299sWBMF4vHVBWpA4F4Gw+55/MwjFCNo1QqED9rKI7NGw43lfQ/olGvDMDsezviqbBgh2/jaAdOr5b+iJk+EOunVL5mh5gzvavzdKtsIm3UXClj/KSa+Hpi8YMhZqX0rfVedW5q/Ef7p4m9oVAoMLpbU7w4uDV+/G9vo+sVCgXaN/aBh5sKwfXthwhrvR6BvuVhx3Risz1uKiW2Th1g/0CRFwdbPuHbop974MyPt3gJeGNfD/z+fB8cfDPcMJnYUv2WYDtDNfPv72JUO8eSsFYN8OtzYYh+baBhKK1v6wZmy79NBfhobA4V2Vvl1S7AG5MHGi+Tt/Qc2zX2xhNhwboVX7bp5yv98XyYxfk6pj9Ss+/tiE8fDUXSvOE4NGsoGvt6YONL/RH92kCzCd6mGtR1/MOMnn7ZvjWhzf2cvk9bdr8+2PC1o71d1YVhhGokDzdVhUvaV1abAG9M6NcSaieW8jozmNSifh2Hn1vbatgd+Kk+wUiYNRTP2Sm/b8nYHs0wzOQTvKWekVn3drSwGqfs9VQpFXh1WDv0Mxnasnefjpp/f2cM6xhgFnYsuUtXkfcxUVut1X2xZtrQttgXORj3dHF8KE7/Cdh0jP6d+zvjzuB6eCqshdnyakvDKg29NZh9b0d89HAIfn7G8nynpHnDbbYlyN9+8OvV0h/BDepgx6t3472xXTDp7lZm7TP1y7O2h6rsLaWfObK92TyiYDubedb1sD2Mos/NPVr4Y8erA82uNw24D/QsG+aqq1Eb9QwGN6iDd+7vjM/HhWKQlXlG+u0z7Nn9+iDD1487WQNnbPdmaFDXvUKBuKmfp9Gwk9wLnRhGiKxwZr5KQ+/qmWU/fXg7TOjbEmsn215W7OzfkXoWhlzuamk8X8bRp6//tLlmch/0bV0fW6b2x3/7tYRapcRjuj+uz91tv2KuWHiHAKPvnRmGauTjga+e7Gkz7OitHN8LG6b0tXkSuMPOiheFQoEmvp7o5sRqK/1JoKHJ6own7mqB3yb1wbzRnc16Hiy9Z0BZD8OY0GZWh9HqatQ2d9W+t0sTs63j9b1nANDUr3yYpVk9Lzx8Z3OjOTqWqJQKu70ydnseFeYhf3jnAIuH6nVs4oMJfVuaFTTUM13JJa4d5F/HHYv+09UojPrYWep6b9dArBjfy+LwijN/E/Q/O6Y/96ZMH+XO4Ho4+GY4xthZjeYIDtMQ3QZGdWmC8X2DsfQx88l6leHt4YbZER3tbuxXmSWHsdMHYf79nc16SvSffHvbKZOv78Xo3rwefpp4l1H5/7dHd8ZfL/bD68OdK5E/Y0Q7vDe2CzRqJRr7eFj8w+5t51OwIzzcVGXd+7a6qB0MZU/3aYnZ93Y0qmsipl+BMqxjgOH5BPl74X8PhuBrKwXbfprYG94eanzySDfHGmHF3zMGWb1OqVTgNZOhmqf6BGPl+DsxsV9LPGKnB0Rs+7S78dPE3tg3c7DdYz3dVXh+oPXeOQXMT773d2uKt0eXBSVLAVehUGB2REc8Y2W7CNOA/dmjZb+v8+7rhPi3wtEx0AdfPtEDKqXC7jCUmKXekU6B5UMu4R2M56mYBsc1z/fByXdGoF4dd7w8pA2eHXCH0Xs+pH0jHLPQw6VUKKBQKMx+RE2Hs759quzny9aSdHurr6rb7Ts1mKiSnFnIo1IqMCeik/0Dq9iyx7sjPbvArHS8M5rX98Lj9c3Hv0OC/HDwzXCrn7gDfT1wObsAQztYnxCoUirsjoNb4uGmwsN3NsfY7s0M96O3YExnbEnKwPi+wU7fryMqOnTurlZigo1JrhP6tsTPz/Q2m8diqzx+39YNcHj2sEp/avX2cMPkga2wNPoMulqYkGrJwHaNzLY6sKd1o7qGeiKVpVAojE6e3ho1FAoFngwLxrhezc12DneEae2cEZ0bI3n+CKP3ZEDbhkh+Z4RT9+8p2mbi8OxhyCsqMQrQj/Zqjo8fCUUddxUS07IQ6OeJ3u/uMFyvVCrgoSxrwytD2xouH92tKQRBsBiW2wV4G1axmQ7hbXqpPyI+341jl3MAAEM6BBhWFQXP3Gh07M8TeyPuwg2zek9SY88IkRVjdScJe7Un5DSicxOM7+v4Kg9nNfTWWJ3hv+HFfvh8XGiF5p44Sq1Smp0UHuvdAj/8t7fRPkM1kaVP7s5OqAWc6z5/5M6ypa2WVqpMDW+Lr5/siR8szKURP4JpMThbxPVzKhLiTG8i/kSvQFnAOzJ3GGaMaIeNL/U3XOdsEHmwRzO0qO+FEZ3MKxBbek+cvf837mmP9o29sfA/XeDr5WYIB+EdAtDUzxN9WzdAXV2YCm1ez2jYzN4cJXEQEUepra8MMEwC9nBTGQ1NKZUKu0Np+rvt07oBXhrSRvZhmpr920wko0A/T5x4e4TFUudUVkPCWp0NAl4f3h4RXQNx72e7AUiz78c793fGgz2DLC7HdVcrDdV1TfVtXT6/xpk6HCM6N8aJt0fgp/0X7M53sERc4G9iv5aY0K8l+izaCaB82MDHww2TBzo/QVPsgwdDrPYwVIUmvp7YYmEV1tdP9oBWMF+yq1Gr8P4DXVFYoq2yqq6mc4rkXh3jLIYRIhuc3WSObj+2hl5sUVZwiKoy3FTKCpV579zUF3+92A9NfD2c/oTs6a6yWa/Glqf7BCMm5SpGdGpseJ23Th2Af/OLHFrl4ww5qosqFApYW5T3UAX2Jpob0Qljlu4x1IkRM33b7D3dmpZVGEaIiKxYM7kPQp1YJWPJmNCmOJyW5XBhPLlIHZyAsp4R02q1lZn/dLsLCfJD8vyRFusvmfa+yFnavSIYRoiIRJr6lS3T9XBTIjTIzkobB3z0cLdqHSIg12KtEKR+ro9+grK9Dq5uQfJtimcJwwgRkYhSqTDUdamqAMEgQtXNz8sdJ94eYZi4aq1C8fZpA7Dm0CU8a2X5s1w4M4+IyIRCV7+BqDbx1O3pBVivKdK6kTdmjGhvtm+P3NgzQkREdJt57u5WqFfH3WxH8ZqKYYSIiOg2465W4nE7m/nVJBymISIiIlkxjBAREZGsGEaIiIhIVgwjREREJCuGESIiIpIVwwgRERHJimGEiIiIZMUwQkRERLJiGCEiIiJZMYwQERGRrBhGiIiISFYMI0RERCQrhhEiIiKSVa3YtVcQBABATk6OzC0hIiIiR+nP2/rzuDW1Iozk5uYCAIKCgmRuCRERETkrNzcXvr6+Vq9XCPbiSg2g1Wpx+fJleHt7Q6FQVNn95uTkICgoCGlpafDx8amy+6WK43tSs/D9qFn4ftQsfD/sEwQBubm5CAwMhFJpfWZIregZUSqVaNasWbXdv4+PD3+Qahi+JzUL34+ahe9HzcL3wzZbPSJ6nMBKREREsmIYISIiIlm5dBjRaDSYM2cONBqN3E0hHb4nNQvfj5qF70fNwvej6tSKCaxERER0+3LpnhEiIiKSH8MIERERyYphhIiIiGTFMEJERESyYhghIiIiWbl0GFmyZAmCg4Ph4eGB3r1748CBA3I36bYQGxuLiIgIBAYGQqFQYN26dUbXC4KA2bNno0mTJvD09ER4eDhOnTpldMy///6Lxx57DD4+PvDz88N///tf5OXlGR1z5MgR9O/fHx4eHggKCsL7779f3U+t1lm4cCHuvPNOeHt7o1GjRrj//vuRnJxsdExBQQFeeOEF1K9fH3Xr1sXYsWORmZlpdExqaipGjRoFLy8vNGrUCNOnT0dJSYnRMdHR0ejevTs0Gg1at26NlStXVvfTq5W++OILdO3a1VC1MywsDJs3bzZcz/dDPosWLYJCocDUqVMNl/H9kIjgolavXi24u7sLy5cvF44dOyY888wzgp+fn5CZmSl302q9TZs2CW+++aawZs0aAYCwdu1ao+sXLVok+Pr6CuvWrRMOHz4s3HfffULLli2FW7duGY4ZMWKEEBISIvzzzz/C33//LbRu3Vp49NFHDddnZ2cLAQEBwmOPPSYkJSUJq1atEjw9PYUvv/xSqqdZKwwfPlxYsWKFkJSUJCQmJgr33HOP0Lx5cyEvL89wzKRJk4SgoCBhx44dQlxcnHDXXXcJffr0MVxfUlIidO7cWQgPDxcSEhKETZs2CQ0aNBAiIyMNx5w9e1bw8vISpk2bJhw/flz47LPPBJVKJWzZskXS51sbbNiwQdi4caOQkpIiJCcnC2+88Ybg5uYmJCUlCYLA90MuBw4cEIKDg4WuXbsKL7/8suFyvh/ScNkw0qtXL+GFF14wfF9aWioEBgYKCxculLFVtx/TMKLVaoXGjRsLH3zwgeGyrKwsQaPRCKtWrRIEQRCOHz8uABAOHjxoOGbz5s2CQqEQLl26JAiCICxdulSoV6+eUFhYaDjm9ddfF9q1a1fNz6h2u3LligBAiImJEQSh7LV3c3MTfvvtN8MxJ06cEAAI+/btEwShLFwqlUohIyPDcMwXX3wh+Pj4GF7/GTNmCJ06dTJ6rIcfflgYPnx4dT+l20K9evWEb775hu+HTHJzc4U2bdoIUVFRwt13320II3w/pOOSwzRFRUWIj49HeHi44TKlUonw8HDs27dPxpbd/s6dO4eMjAyj197X1xe9e/c2vPb79u2Dn58fevbsaTgmPDwcSqUS+/fvNxwzYMAAuLu7G44ZPnw4kpOTcePGDYmeTe2TnZ0NAPD39wcAxMfHo7i42Oj9aN++PZo3b270fnTp0gUBAQGGY4YPH46cnBwcO3bMcIz4PvTH8PfJttLSUqxevRr5+fkICwvj+yGTF154AaNGjTJ7zfh+SKdW7Npb1a5du4bS0lKjHx4ACAgIwMmTJ2VqlWvIyMgAAIuvvf66jIwMNGrUyOh6tVoNf39/o2Natmxpdh/66+rVq1ct7a/NtFotpk6dir59+6Jz584Ayl4rd3d3+Pn5GR1r+n5Yer/019k6JicnB7du3YKnp2d1PKVa6+jRowgLC0NBQQHq1q2LtWvXomPHjkhMTOT7IbHVq1fj0KFDOHjwoNl1/P2QjkuGESJX9MILLyApKQm7d++Wuykur127dkhMTER2djZ+//13PPXUU4iJiZG7WS4nLS0NL7/8MqKiouDh4SF3c1yaSw7TNGjQACqVymxGdGZmJho3bixTq1yD/vW19do3btwYV65cMbq+pKQE//77r9Exlu5D/BhUbsqUKfjrr7+wa9cuNGvWzHB548aNUVRUhKysLKPjTd8Pe6+1tWN8fHz4qc8Cd3d3tG7dGj169MDChQsREhKCTz75hO+HxOLj43HlyhV0794darUaarUaMTEx+PTTT6FWqxEQEMD3QyIuGUbc3d3Ro0cP7Nixw3CZVqvFjh07EBYWJmPLbn8tW7ZE48aNjV77nJwc7N+/3/Dah4WFISsrC/Hx8YZjdu7cCa1Wi969exuOiY2NRXFxseGYqKgotGvXjkM0IoIgYMqUKVi7di127txpNrTVo0cPuLm5Gb0fycnJSE1NNXo/jh49ahQQo6Ki4OPjg44dOxqOEd+H/hj+PjlGq9WisLCQ74fEhgwZgqNHjyIxMdHwr2fPnnjssccMX/P9kIjcM2jlsnr1akGj0QgrV64Ujh8/Ljz77LOCn5+f0Yxoqpjc3FwhISFBSEhIEAAIH374oZCQkCBcuHBBEISypb1+fn7C+vXrhSNHjgijR4+2uLQ3NDRU2L9/v7B7926hTZs2Rkt7s7KyhICAAOGJJ54QkpKShNWrVwteXl5c2mvi+eefF3x9fYXo6GghPT3d8O/mzZuGYyZNmiQ0b95c2LlzpxAXFyeEhYUJYWFhhuv1SxeHDRsmJCYmClu2bBEaNmxoceni9OnThRMnTghLlizh0kUrZs6cKcTExAjnzp0Tjhw5IsycOVNQKBTCtm3bBEHg+yE38WoaQeD7IRWXDSOCIAifffaZ0Lx5c8Hd3V3o1auX8M8//8jdpNvCrl27BABm/5566ilBEMqW986aNUsICAgQNBqNMGTIECE5OdnoPq5fvy48+uijQt26dQUfHx9h/PjxQm5urtExhw8fFvr16ydoNBqhadOmwqJFi6R6irWGpfcBgLBixQrDMbdu3RImT54s1KtXT/Dy8hLGjBkjpKenG93P+fPnhZEjRwqenp5CgwYNhFdffVUoLi42OmbXrl1Ct27dBHd3d+GOO+4wegwqN2HCBKFFixaCu7u70LBhQ2HIkCGGICIIfD/kZhpG+H5IQyEIgiBPnwwRERGRi84ZISIiopqDYYSIiIhkxTBCREREsmIYISIiIlkxjBAREZGsGEaIiIhIVgwjREREJCuGESIiIpIVwwgRERHJimGEiIiIZMUwQkRERLL6PwlYV6JZtiU9AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot([p[\"loss\"] for p in data[\"log_history\"]])\n", + "plt.title(\"Loss\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "dl3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.8" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "90bfda469df5ac7fed8d7e225d563f60a7a7aa420ccfadb091c914debf775e49" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}