{"cells":[{"cell_type":"markdown","metadata":{"id":"Q-bj6K7Qv4ft"},"source":["# Fine-Tuning a Generative Pretrained Transformer (`GPT`)\n","\n","1. Install required libraries."]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":16079,"status":"ok","timestamp":1687456413307,"user":{"displayName":"Nicholas Corrêa","userId":"09736120585766268588"},"user_tz":-120},"id":"SBWCrz5GfBXo","outputId":"0d3078fb-2d35-4fb2-b56e-6bed6ea2d895"},"outputs":[{"name":"stdout","output_type":"stream","text":["Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Collecting transformers\n","  Downloading transformers-4.30.2-py3-none-any.whl (7.2 MB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.2/7.2 MB\u001b[0m \u001b[31m57.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.0)\n","Collecting huggingface-hub<1.0,>=0.14.1 (from transformers)\n","  Downloading huggingface_hub-0.15.1-py3-none-any.whl (236 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m236.8/236.8 kB\u001b[0m \u001b[31m28.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.22.4)\n","Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.1)\n","Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0)\n","Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2022.10.31)\n","Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.27.1)\n","Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers)\n","  Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m103.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting safetensors>=0.3.1 (from transformers)\n","  Downloading safetensors-0.3.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m75.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.65.0)\n","Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (2023.4.0)\n","Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (4.5.0)\n","Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (1.26.15)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2022.12.7)\n","Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.12)\n","Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4)\n","Installing collected packages: tokenizers, safetensors, huggingface-hub, transformers\n","Successfully installed huggingface-hub-0.15.1 safetensors-0.3.1 tokenizers-0.13.3 transformers-4.30.2\n","Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Collecting codecarbon\n","  Downloading codecarbon-2.2.4-py3-none-any.whl (176 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m176.0/176.0 kB\u001b[0m \u001b[31m5.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting arrow (from codecarbon)\n","  Downloading arrow-1.2.3-py3-none-any.whl (66 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m66.4/66.4 kB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from codecarbon) (1.5.3)\n","Collecting pynvml (from codecarbon)\n","  Downloading pynvml-11.5.0-py3-none-any.whl (53 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.1/53.1 kB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from codecarbon) (2.27.1)\n","Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from codecarbon) (5.9.5)\n","Requirement already satisfied: py-cpuinfo in /usr/local/lib/python3.10/dist-packages (from codecarbon) (9.0.0)\n","Collecting fuzzywuzzy (from codecarbon)\n","  Downloading fuzzywuzzy-0.18.0-py2.py3-none-any.whl (18 kB)\n","Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from codecarbon) (8.1.3)\n","Requirement already satisfied: python-dateutil>=2.7.0 in /usr/local/lib/python3.10/dist-packages (from arrow->codecarbon) (2.8.2)\n","Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->codecarbon) (2022.7.1)\n","Requirement already satisfied: numpy>=1.21.0 in /usr/local/lib/python3.10/dist-packages (from pandas->codecarbon) (1.22.4)\n","Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->codecarbon) (1.26.15)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->codecarbon) (2022.12.7)\n","Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->codecarbon) (2.0.12)\n","Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->codecarbon) (3.4)\n","Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.7.0->arrow->codecarbon) (1.16.0)\n","Installing collected packages: fuzzywuzzy, pynvml, arrow, codecarbon\n","Successfully installed arrow-1.2.3 codecarbon-2.2.4 fuzzywuzzy-0.18.0 pynvml-11.5.0\n"]}],"source":["!pip install transformers\n","!pip install codecarbon"]},{"cell_type":"code","execution_count":3,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":17938,"status":"ok","timestamp":1687474615915,"user":{"displayName":"Nicholas Corrêa","userId":"09736120585766268588"},"user_tz":-120},"id":"kU_MRV-QB7CN","outputId":"517e66a1-5a4f-4a19-b2a8-b62642dcaf1e"},"outputs":[{"name":"stdout","output_type":"stream","text":["Mounted at /content/drive\n"]}],"source":["from google.colab import drive\n","drive.mount('/content/drive')"]},{"cell_type":"markdown","metadata":{"id":"y5XnfvSH7w4z"},"source":["2. Load the data from the hub."]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":423},"executionInfo":{"elapsed":1483,"status":"ok","timestamp":1687456464404,"user":{"displayName":"Nicholas Corrêa","userId":"09736120585766268588"},"user_tz":-120},"id":"7MbpXGu-v4f1","outputId":"5971618b-9626-4253-c325-609ea639cb3e"},"outputs":[{"data":{"text/html":["\n","  <div id=\"df-9b9de4dc-acff-4a79-a1d2-967a4ad96ddc\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>prompt</th>\n","      <th>completion</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Which is a species of fish? Tope or Rope</td>\n","      <td>Tope</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Why can camels survive for long without water?</td>\n","      <td>Camels use the fat in their humps to keep them...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Alice's parents have three daughters: Amy, Jes...</td>\n","      <td>The name of the third daughter is Alice</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Who gave the UN the land in NY to build their HQ</td>\n","      <td>John D Rockerfeller</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Why mobile is bad for human</td>\n","      <td>We are always engaged one phone which is not g...</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>53129</th>\n","      <td>How do computers communicate and network with ...</td>\n","      <td>Computers communicate and network with each ot...</td>\n","    </tr>\n","    <tr>\n","      <th>53130</th>\n","      <td>How are websites different from web applications?</td>\n","      <td>Websites and web applications are similar in t...</td>\n","    </tr>\n","    <tr>\n","      <th>53131</th>\n","      <td>What is open-source software and its benefits?</td>\n","      <td>Open-source software is software that is made ...</td>\n","    </tr>\n","    <tr>\n","      <th>53132</th>\n","      <td>What is a cookie and how is it used in web bro...</td>\n","      <td>A cookie is a small piece of data that a websi...</td>\n","    </tr>\n","    <tr>\n","      <th>53133</th>\n","      <td>What is cloud storage and its advantages for d...</td>\n","      <td>Cloud storage is a service that allows you to ...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>53134 rows × 2 columns</p>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-9b9de4dc-acff-4a79-a1d2-967a4ad96ddc')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-9b9de4dc-acff-4a79-a1d2-967a4ad96ddc button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-9b9de4dc-acff-4a79-a1d2-967a4ad96ddc');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "],"text/plain":["                                                  prompt  \\\n","0               Which is a species of fish? Tope or Rope   \n","1         Why can camels survive for long without water?   \n","2      Alice's parents have three daughters: Amy, Jes...   \n","3       Who gave the UN the land in NY to build their HQ   \n","4                            Why mobile is bad for human   \n","...                                                  ...   \n","53129  How do computers communicate and network with ...   \n","53130  How are websites different from web applications?   \n","53131     What is open-source software and its benefits?   \n","53132  What is a cookie and how is it used in web bro...   \n","53133  What is cloud storage and its advantages for d...   \n","\n","                                              completion  \n","0                                                   Tope  \n","1      Camels use the fat in their humps to keep them...  \n","2                The name of the third daughter is Alice  \n","3                                    John D Rockerfeller  \n","4      We are always engaged one phone which is not g...  \n","...                                                  ...  \n","53129  Computers communicate and network with each ot...  \n","53130  Websites and web applications are similar in t...  \n","53131  Open-source software is software that is made ...  \n","53132  A cookie is a small piece of data that a websi...  \n","53133  Cloud storage is a service that allows you to ...  \n","\n","[53134 rows x 2 columns]"]},"metadata":{},"output_type":"display_data"}],"source":["import pandas as pd\n","from datasets import load_dataset\n","\n","dataset = load_dataset(\"nicholasKluge/fine-tuning-instruct-aira\", split=\"aira_instruct_english\")\n","\n","df = dataset.to_pandas()\n","\n","display(df)"]},{"cell_type":"markdown","metadata":{"id":"fEaDTvdOe8rr"},"source":["3. Load `GPT2Tokenizer` and add the chosen special tokens (`'<|startoftext|>', '<|endoftext|>','<|pad|>'`)\n","4. Create demonstrations by prepending the special tokens.\n","5. Calculate the maximum length (in tokens) that the demonstrations have (the dataset was constructed, for efficiency and fast training, to be below the 300-token range)."]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":185,"referenced_widgets":["3810e8894371460f99bae37100f6bbf3","5f4ba651d3ff4a6caa7c58ae6a5765d0","4c0621b826c543a3bf4b4814d49b03ca","69490541eed442599ce7407614edcfcb","6479e02a6d3c4681848af06e62d0662e","03292a675acf4db6947d6701d23fda8f","56e10d229ff242c795b686e05e9350ad","089a2b41007446be84c9face82cce498","c57fadd502b048dca7d250e178ebbbe1","39b9b8e6871a43b3b87cb9e14672df16","82be6e79314d42219d7823d98636c187","ca9259567e0f4e97aac6f83026431015","07673191fca947e38ee294794f81eb9f","2affe04d18994a99882b170e237bf6c6","71cca7b8799649d4a876875eda9116c0","378582eca5544619a05a6acc62c0e7e0","7b724b134e494fd0af9359f9c1814f9f","3f4a13cbdf0f4c148c03c5ca1cf416c5","10ce7e44cdd745c9bfe19f1fecd3a327","5a0cd808c80f4a189068851b9294147e","29d9a194f848426cb340ae0cc74c9a8e","8f4bec13c6864463b85e5615332a165b","c683803c7d6c4f8f9aef51120b6de2a4","dc510021f0454762a27ee6ad0ab7c843","cf58770b9bdc42f0bc7024bb3e5b1b0a","07e0a69209424a36af75143ffe24ce24","a52ae49eabbc485dba35f93214118663","ae609ee3e0444ec491e080ded8e6bc0d","1157a3061d344f25ae3433fcb07e692c","cbf3d77427bf4760b7bbffd7f5433449","8b01dd30e9fe4cef991299b36afb7c2f","85667b6eb2764e859b3e4daf67f2922f","73e56ce685b34d868e88f7feae113b73"]},"executionInfo":{"elapsed":35769,"status":"ok","timestamp":1687456503345,"user":{"displayName":"Nicholas Corrêa","userId":"09736120585766268588"},"user_tz":-120},"id":"hfu84fWIv4f9","outputId":"1199d785-5210-444e-dfc3-a6c20811a87e"},"outputs":[{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"3810e8894371460f99bae37100f6bbf3","version_major":2,"version_minor":0},"text/plain":["Downloading (…)olve/main/vocab.json: 0.00B [00:00, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"ca9259567e0f4e97aac6f83026431015","version_major":2,"version_minor":0},"text/plain":["Downloading (…)olve/main/merges.txt: 0.00B [00:00, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"c683803c7d6c4f8f9aef51120b6de2a4","version_major":2,"version_minor":0},"text/plain":["Downloading (…)lve/main/config.json:   0%|          | 0.00/689 [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stderr","output_type":"stream","text":["Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"]},{"name":"stdout","output_type":"stream","text":["Total number of demonstrations:  53134\n","The longest demonstration is 300 tokens long.\n"]}],"source":["from transformers import GPT2Tokenizer\n","\n","model = \"gpt2-xl\" # \"gpt2\", \"gpt2-medium\", \"gpt2-large\", \"gpt2-xl\", \"pierreguillou/gpt2-small-portuguese\"\n","model_size = \"1.5B\" # \"124M\", \"355M\", \"774M\", and \"1.5B\", \"PT-124M\"\n","\n","tokenizer = GPT2Tokenizer.from_pretrained(model,\n","                                          bos_token='<|startoftext|>',\n","                                          eos_token='<|endoftext|>',\n","                                          pad_token='<|pad|>')\n","\n","df['demonstrations'] = tokenizer.bos_token + df['prompt'] + tokenizer.eos_token + df['completion'] + tokenizer.eos_token\n","\n","df['length'] = df['demonstrations'].apply(lambda x: len(tokenizer.encode(x)))\n","\n","print(\"Total number of demonstrations: \", len(df))\n","print(f\"The longest demonstration is {df['length'].max()} tokens long.\")"]},{"cell_type":"markdown","metadata":{"id":"wkMO17K5e8rs"},"source":["6. Create the Dataset class."]},{"cell_type":"code","execution_count":null,"metadata":{"id":"WlbAfMQ4v4gA"},"outputs":[],"source":["import torch\n","from torch.utils.data import Dataset\n","\n","max_length = 300\n","\n","class DemoDataset(Dataset):\n","\n","  def __init__(self, demonstrations, tokenizer, gpt2_type=\"gpt2\", max_length=max_length):\n","\n","    self.tokenizer = tokenizer\n","    self.input_ids = []\n","    self.attn_masks = []\n","\n","    for demo in demonstrations:\n","\n","      encodings_dict = tokenizer(demo,\n","                                 truncation=True,\n","                                 max_length=max_length,\n","                                 padding=\"max_length\")\n","\n","      self.input_ids.append(torch.tensor(encodings_dict['input_ids']))\n","      self.attn_masks.append(torch.tensor(encodings_dict['attention_mask']))\n","\n","  def __len__(self):\n","    return len(self.input_ids)\n","\n","  def __getitem__(self, idx):\n","    return self.input_ids[idx], self.attn_masks[idx]"]},{"cell_type":"markdown","metadata":{"id":"711Krm6Te8rt"},"source":["7. Split the data into training and validation splits."]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":40129,"status":"ok","timestamp":1687456550403,"user":{"displayName":"Nicholas Corrêa","userId":"09736120585766268588"},"user_tz":-120},"id":"-IOfa2PEv4gD","outputId":"3d0b9a15-5f52-48eb-bc3f-d092f483b12d"},"outputs":[{"name":"stdout","output_type":"stream","text":["Number of training samples: 47,820\n","Number of validation samples: 5,314\n"]}],"source":["from torch.utils.data import random_split\n","\n","dataset = DemoDataset(df.demonstrations.to_list(), tokenizer, max_length=max_length)\n","\n","train_size = int(0.9 * len(dataset))\n","val_size = len(dataset) - train_size\n","\n","train_dataset, val_dataset = random_split(dataset, [train_size, val_size])\n","\n","print('Number of training samples: {:,}'.format(train_size))\n","print('Number of validation samples: {:,}'.format(val_size))"]},{"cell_type":"markdown","metadata":{"id":"vSqKuRjIe8ru"},"source":["8. Create the `DataLoaders` and specify the `batch_size`."]},{"cell_type":"code","execution_count":null,"metadata":{"id":"cUkCNV-6v4gG"},"outputs":[],"source":["from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n","\n","train_dataloader = DataLoader(\n","            train_dataset,\n","            sampler=RandomSampler(train_dataset),\n","            batch_size=4 # 32, 20, 8, 4\n","        )\n","\n","# validation data loader doesn't need randomization\n","validation_dataloader=DataLoader(\n","            val_dataset,\n","            sampler=SequentialSampler(val_dataset),\n","            batch_size=4\n","        )"]},{"cell_type":"markdown","metadata":{"id":"0vxvcTIHe8rv"},"source":["9. Load the base model (`GPT2LMHeadModel`)."]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":549,"referenced_widgets":["9c806d69e76642d0b6f023f8651ccd0f","613675b2de024ff0b16d73e2d83d3ad3","1802795ff27a45fb89f2f1cc9d670efa","9a3c9e1a67c1495fa09a1d2900b0dc04","add94d96c6c24c0ca6cfc2402331b661","0008ec0128a74eaba3678973d123af9a","09ebe3ce4af048da8c89f33fc8735080","657aab3bbcd9441b8d1eeed0304e86a3","ce831862e4e94a8aa0918f2158d67392","805d2f78d9d44927bea32887f797b114","a66352b3c2d044538e3c6ed950e2cbfb","15115eecc97c4b719f482fbd03971627","d76adfa0730b4e10be62eb71a169a324","576e5200ec394e96b118e2ad8284334e","417458df670b47478df012d9a8dd83f0","48e8ea41008b402291fde670c9b0f838","432ac604f66245a7b859a76915718b99","ba7059e2f9b54e4c8dac069bf8bdd503","bf19c1ff40d641cba4ea07e51203c6cf","c1a3598e009b421c8799860a13b320f4","f569465326a642e898847d3929daa508","d80df8cc256742df96ac16847523f11d"]},"executionInfo":{"elapsed":146187,"status":"ok","timestamp":1687456696574,"user":{"displayName":"Nicholas Corrêa","userId":"09736120585766268588"},"user_tz":-120},"id":"Rmg-5YJqv4gH","outputId":"a87bc9ef-c28b-4629-d3aa-131d64887780"},"outputs":[{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"9c806d69e76642d0b6f023f8651ccd0f","version_major":2,"version_minor":0},"text/plain":["Downloading pytorch_model.bin:   0%|          | 0.00/6.43G [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"15115eecc97c4b719f482fbd03971627","version_major":2,"version_minor":0},"text/plain":["Downloading (…)neration_config.json:   0%|          | 0.00/124 [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"text/plain":["GPT2LMHeadModel(\n","  (transformer): GPT2Model(\n","    (wte): Embedding(50259, 1600)\n","    (wpe): Embedding(1024, 1600)\n","    (drop): Dropout(p=0.1, inplace=False)\n","    (h): ModuleList(\n","      (0-47): 48 x GPT2Block(\n","        (ln_1): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n","        (attn): GPT2Attention(\n","          (c_attn): Conv1D()\n","          (c_proj): Conv1D()\n","          (attn_dropout): Dropout(p=0.1, inplace=False)\n","          (resid_dropout): Dropout(p=0.1, inplace=False)\n","        )\n","        (ln_2): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n","        (mlp): GPT2MLP(\n","          (c_fc): Conv1D()\n","          (c_proj): Conv1D()\n","          (act): NewGELUActivation()\n","          (dropout): Dropout(p=0.1, inplace=False)\n","        )\n","      )\n","    )\n","    (ln_f): LayerNorm((1600,), eps=1e-05, elementwise_affine=True)\n","  )\n","  (lm_head): Linear(in_features=1600, out_features=50259, bias=False)\n",")"]},"execution_count":8,"metadata":{},"output_type":"execute_result"}],"source":["from transformers import GPT2LMHeadModel, GPT2Config\n","\n","configuration = GPT2Config.from_pretrained(model, output_hidden_states=False)\n","\n","model = GPT2LMHeadModel.from_pretrained(model, config=configuration)\n","model.resize_token_embeddings(len(tokenizer))\n","\n","device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n","\n","model.to(device)"]},{"cell_type":"markdown","metadata":{"id":"XAoXD7GVYMN_"},"source":["10. Freeze some of the layers for constrained fine-tuning. This allows the model to retain some of its original capabilities after the tuning."]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":31,"status":"ok","timestamp":1687456696575,"user":{"displayName":"Nicholas Corrêa","userId":"09736120585766268588"},"user_tz":-120},"id":"MJA-WOoHZePn","outputId":"fee8522b-2a2d-4ada-cf42-c0f86737686e"},"outputs":[{"name":"stdout","output_type":"stream","text":["Number of transformer blocks in the model:  48\n","Number of transformer blocks to  un-freeze:  12\n","Number of frozen layers: 433\n","Number of trainable layers: 147\n"]}],"source":["UNFREEZE_LAST_N = 12 #6, 8, 9, 12\n","\n","print(\"Number of transformer blocks in the model: \", model.config.n_layer)\n","print(\"Number of transformer blocks to  un-freeze: \", UNFREEZE_LAST_N)\n","\n","for parameter in model.parameters():\n","    parameter.requires_grad = False\n","\n","for i, m in enumerate(model.transformer.h):\n","    #Only un-freeze the last n transformer blocks\n","    if i+1 > model.config.n_layer - UNFREEZE_LAST_N:\n","        for parameter in m.parameters():\n","            parameter.requires_grad = True\n","\n","        for parameter in model.transformer.ln_f.parameters():\n","            parameter.requires_grad = True\n","\n","        for parameter in model.lm_head.parameters():\n","            parameter.requires_grad = True\n","\n","num_frozen_layers = sum(1 for parameter in model.parameters() if not parameter.requires_grad)\n","num_trainable_layers = sum(1 for parameter in model.parameters() if parameter.requires_grad)\n","\n","print(\"Number of frozen layers:\", num_frozen_layers)\n","print(\"Number of trainable layers:\", num_trainable_layers)"]},{"cell_type":"markdown","metadata":{"id":"GJ1kH3uSe8rw"},"source":["11. Set the training hyperparameters."]},{"cell_type":"code","execution_count":null,"metadata":{"id":"qlbLg6tqv4gI"},"outputs":[],"source":["from transformers import get_linear_schedule_with_warmup\n","\n","# training rounds\n","epochs = 2 # 5, 3, 2, 2\n","\n","# steps at the start of training that are ignored\n","warmup_steps = 1e2\n","\n","# sample the model to test the output\n","sample_every = 400\n","\n","optimizer = torch.optim.AdamW(model.parameters(), lr = 5e-4, eps = 1e-8)\n","\n","# total steps = number of batches * number of epochs\n","total_steps = len(train_dataloader) * epochs\n","\n","# create the learning rate scheduler\n","scheduler = get_linear_schedule_with_warmup(optimizer,\n","                                            num_warmup_steps = warmup_steps,\n","                                            num_training_steps = total_steps)"]},{"cell_type":"markdown","metadata":{"id":"nz1a9lxXe8rw"},"source":["12. Training/Validation loop. Track the carbon emissions of your work by using `codecarbon`. 🌱"]},{"cell_type":"code","execution_count":11,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":408317,"status":"ok","timestamp":1687474409683,"user":{"displayName":"Nicholas Corrêa","userId":"09736120585766268588"},"user_tz":-120},"id":"_X_m8XOtv4gR","outputId":"7ee113ab-3b16-4b45-9246-d08b1ab28f6b"},"outputs":[{"name":"stdout","output_type":"stream","text":["\n","Beginning epoch 1 of 2\n","\n"]},{"name":"stderr","output_type":"stream","text":["  3%|▎         | 400/11955 [04:41<2:13:56,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 400 of 11955. Loss:1.1654434204101562.\n","\n","\n","Example output: What is open-source software and its benefits?Computer science is the subfield of computer science that examines the intricate relationship between machines and humans, and examines how such relationships should be created and modified. It is essential for today's technology environment, and plays a crucial role in creating technologies that are ever more accessible, responsive, and responsive. In addition, machine learning can be utilized to solve intricate problems, such as creating AI tools that can recognize and respond appropriately to human emotions or emotions themselves. This enables users to express themselves and respond to emotions themselves, which is a big area of focus for artificial intelligence, where machine learning is utilized to learn and respond appropriately to emotions itself, and this is where deep learning is utilized to model and respond appropriately to emotions themselves. Machine learning is utilized for a wide variety of tasks, such as generating captions, generating speech recognition, generating text, generating text generation, generating text translation, generating text summarization, generating text summarization and speech recognition,\n","\n"]},{"name":"stderr","output_type":"stream","text":["  7%|▋         | 800/11955 [09:31<2:09:46,  1.43it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 800 of 11955. Loss:0.6719180345535278.\n","\n","\n","Example output: I am writing an academic essay on the economy, can you provide some sources or references to help me with my research.Sure! A computer program was developed to analyze the effects of the war on the economy and the economy was monitored while the war was ongoing. It found that the economy as a whole was relatively stable during the war and was relatively positive afterwards. However, some businesses saw greater returns on their investment during the war and it is possible that some of the businesses benefited because they had the right equipment or they had a good idea. Other businesses may have become more successful during the war due to the need for more resources to be available while still others may have become more successful for other reasons.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 10%|█         | 1200/11955 [14:17<2:04:41,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 1200 of 11955. Loss:0.5023787617683411.\n","\n","\n","Example output: What is the role of convolutional neural network in machine learning?In order to learn more about the relationship between neural networks and machine learning, I suggest you start by reading the Neural Network Labeling and Local Area Neural Network (LNN) Neural Network: A Guide for Students and developers. The complete study and explanation is freely available online at Udemy, so you could easily find it online. It is a great overview of the concepts and practices behind NNNs and LNNs and is an easy-to-read and easy-to-read-and-read-and-read-and-read-and-read-and-read-and-read-and-read-and-read-and-read-and-read-and-read-and-read-and-read-and-read-and-read-and-read-and-read-and-read-and-read-and-read-and-read-and\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 13%|█▎        | 1600/11955 [19:05<2:00:07,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 1600 of 11955. Loss:1.0423882007598877.\n","\n","\n","Example output: What are some ways to tour the canals of Amsterdam.The canals of Amsterdam are well developed, but they get more amenities and amenities require more resources. Depending on the size of the lake and the size of the lake beds, they can be able to accommodate many more amenities than other larger lake parks in the Netherlands. While other lake parks in the Netherlands have great amenities, they tend to be larger. You will typically get better value for money in the smaller Lake Canals of Amsterdam, as they tend to have shorter trip times and smaller entrance fee costs, as well as a lower rate of maintenance.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 17%|█▋        | 2000/11955 [23:50<1:55:30,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 2000 of 11955. Loss:0.8089869022369385.\n","\n","\n","Example output: What causes ocean tides?Causes of ocean tides include increasing energy from the Earth's winds, as well as from other atmospheric dynamics. As the waves form, they must move from one location to another in the ocean and the oceans and their habitats need to handle the increased energy from the winds. As the waves form, they must use energy from the Earth's winds and from other atmospheric dynamics to create movement and momentum in the water. As the waves form and move towards each other, they gain speed and power, which causes them to bend the water around their edges. These waves form into a path that is characterized by a peak and a decline. The peak is the point at which the water flows forwards. The decline is the point at which the water flows backward. This creates the visible path from point of peak to point of decline. As the waves move towards each other, they gain momentum and energy. When the waves reach the end of their path, their velocity increases and they fly\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 20%|██        | 2400/11955 [28:39<1:50:51,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 2400 of 11955. Loss:0.3757154941558838.\n","\n","\n","Example output: What is the difference between a meteor, an asteroid and a comet.The main difference between a meteor, an asteroid, and a comet is that a meteor is an tiny bits of dust and dust dust particles, while an Asteroo is a massive cloud of debris, dust, and gas. Asteroids are more likely to burn up in the atmosphere, while objects in the Solar System are less likely to burn up.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 23%|██▎       | 2800/11955 [33:22<1:46:11,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 2800 of 11955. Loss:0.5319542288780212.\n","\n","\n","Example output: What are the symptoms of a UTI.The most common symptoms of a urinary tract infection (UTI) include cloudy or bloody urine, increased urinary frequency, burning sensation, abdominal discomfort, abdominal pain, abdominal cramps, heartburn, and fever. Other signs may include pelvic discomfort and pelvic discomfort and pelvic discomfort and abdominal cramps.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 27%|██▋       | 3200/11955 [38:04<1:41:38,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 3200 of 11955. Loss:0.7579741477966309.\n","\n","\n","Example output: What are the best tips for starting a small business.The best tips for starting a small business include researching, having a plan and staying organized. Start by researching your business objectives and values. Consider what resources available such as mentors, mentors who can help you develop your plan and how you can pay for it. Make sure your plan is realistic and achievable. You may also need to make changes to your plan as needed. Once you have your plan, make sure you focus your effort and focus on making sure your goals are met. Additionally, develop the right marketing strategy to help your business grow and reach your target demographic. Finally, make sure your company is up to date on its security protocols and management procedures.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 30%|███       | 3600/11955 [42:50<1:36:59,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 3600 of 11955. Loss:0.5735041499137878.\n","\n","\n","Example output: What are some tips for maintaining mental health.Eating a balanced diet is one of the key components of maintaining mental state. To ensure that you have the necessary nutrients to maintain a balanced diet, it is important to plan your diet around the nutritious foods that you are exposed to. You should listen to your body and drink plenty of water to stay hydrated. Additionally, practice relaxation techniques such as meditation and deep breathing to reduce stress and increase your energy levels. Lastly, it is important to seek out support from people who are supportive of you and know your individual needs.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 33%|███▎      | 4000/11955 [47:35<1:32:17,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 4000 of 11955. Loss:0.5229397416114807.\n","\n","\n","Example output: What are the components of a business plan.The components of a business plan include a mission statement, a mission and goals, financial projections, a market plan, a funding solution, and an organizational structure.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 37%|███▋      | 4400/11955 [52:15<1:27:42,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 4400 of 11955. Loss:0.7181077599525452.\n","\n","\n","Example output: How do you define a utility function?There are many potential interpretations of the utility function, but one important consideration is the idea of value complexity. This means that when looking at the utility function, consider whether you're looking at the most complex alternatives possible. If you are looking at the most complex options possible, you should consider whether it is possible to solve for all possible values of a parameter by consulting an algorithm. If it is not possible to solve for all possible values by consulting an algorithm, then you should consider whether you are considering all possible alternatives.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 40%|████      | 4800/11955 [56:59<1:22:58,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 4800 of 11955. Loss:0.5860748887062073.\n","\n","\n","Example output: I need to replace the tires on my car, how much should I expect to pay.You should pay the minimum price for the car models you plan to purchase. Generally, you should pay for an electric car and a gas car at the same prices. This allows for an easier comparison, since most electric vehicles are currently priced at the same rate as gasoline-powered vehicles.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 43%|████▎     | 5200/11955 [1:01:41<1:18:23,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 5200 of 11955. Loss:0.5548998713493347.\n","\n","\n","Example output: When was the NFL founded? Which NFL team won the most Super Bowls? Which quarterback has won the most Super Bowls?The National Football League was founded on August 15, 1940 in Seattle, WA, USA. According to the National Football League’s website, the initial goal of the league was to meet the demand for football among American adults, with the intention of expanding into other sports like volleyball, rugby, soccer and handball, among others. The initial success of the league in this regard in 1950 (19,867 athletes), put the name of the league on the table in 1960, when a vote was held to name the league after the first name the first owner of the Baltimore Baltimore Baltimore Baltimore Baltimore Baltimore Baltimore Baltimore franchise would take as the name of the franchise. The name of the first owner was Walter Washington, the founder of the Baltimore Baltimore Baltimore Baltimore Baltimore Baltimore Baltimore Baltimore Baltimore Baltimore Baltimore Baltimore Baltimore Baltimore Baltimore Baltimore Baltimore Baltimore Baltimore Baltimore Baltimore Baltimore Baltimore Baltimore Baltimore\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 47%|████▋     | 5600/11955 [1:06:29<1:13:42,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 5600 of 11955. Loss:0.5516342520713806.\n","\n","\n","Example output: How do I make shortbread cookies.Start by preheating your oven to 350° F (180° C).\n","2  Roll out the cookie dough and fold it into half. Place the folded dough into the bottom of a 9-inch (23-cm) 9-inch (23-cm) cookie sheet.\n","3  Pour the melted butter into a 9-inch (23-cm) 9-inch (23-cm) cookie sheet.\n","4  Top with a 9-inch (23-cm) 9-inch (23-cm) cookie sheet.\n","5  Flatten the bottom of the dough with the bottom of the cookie sheet.\n","6  Flatten the top of the dough with the top of the cookie sheet.\n","7  Bake for 15-20 minutes.\n","8  Cool for 10 minutes.\n","9  Allow the cookies to cool for 10 minutes before serving. Enjoy.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 50%|█████     | 6000/11955 [1:11:18<1:09:03,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 6000 of 11955. Loss:0.6980631351470947.\n","\n","\n","Example output: How can I learn something new every day.One way to learn new skills or knowledge every day is to practice it at home. Start with small projects such as writing, drawing, writing and drawing games. Practice new skills every day and gradually build on them. You will become comfortable with the new skills and be more comfortable with the concepts they are trying to solve. Additionally, you will learn the concepts related to the new skill, so you will also be able to use them while writing, drawing, writing and drawing games. You may also want to start studying resources like tutorials, apps and online resources such as tutorials and apps that can help you learn new skills. Finally, you should also take regular breaks to rest and recharge, as well as be consistent in your practice and focus on understanding the concepts and concepts you are trying to learn.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 54%|█████▎    | 6400/11955 [1:16:06<1:04:25,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 6400 of 11955. Loss:0.4096757769584656.\n","\n","\n","Example output: Is it wise to invest all of one's money in cryptocurrency?It depends. Cryptocurrency is currently an alternative asset asset class to traditional investments such as stocks and bonds, and there are potential risks and rewards. Cryptocurrency is known for providing users with access to the markets that have typically been inaccessible or difficult to access for residents of certain countries. Furthermore, as a blockchain-based technology, Cryptocurrency carries greater value than what it stands as today. Ultimately, the decision of which cryptocurrencies is right for you should depend on your financial goals, timeline, and risk appetite.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 57%|█████▋    | 6800/11955 [1:20:50<59:46,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 6800 of 11955. Loss:0.768928050994873.\n","\n","\n","Example output: How do I archive old emails.Start by creating a new filing cabinet or sorting list on your computer. This will help you keep all of your important emails and programs in one place. Next, install delete-on-read-only software on your computer. This will prevent any emails from being stored, delete any emails from your computer, and block any from getting in your computer or getting in your browser. Make sure to scan your computer for any broken or suspicious software, and remove any that can be found. You may also need to replace any RAM or hard drive that could be causing issues. After the issue is patched, open the file management settings of your computer and delete any unnecessary files. Finally, clean up any clutter that may have built up.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 60%|██████    | 7200/11955 [1:25:37<55:09,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 7200 of 11955. Loss:0.7014164328575134.\n","\n","\n","Example output: What is the perfect place for my family to vacation this summer.The best place for your family to vacation this summer is the ocean. With so many exciting activities for a couple of family members, you will be sure to see a variety of places and attractions. A warm summer evening could be made up of visiting the local bars, taking the stairs to the upper floor of a building, and cooking a delicious meal in a food storage room. For an opportunity to experience all aspects of the summer, visiting an amusement park could be a great option. Whether it's going to a different outdoor activity every day of the week or even visiting the amusement park on a weekends, the most popular places to visit for your family are the beach and beach vacations.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 64%|██████▎   | 7600/11955 [1:30:23<50:30,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 7600 of 11955. Loss:0.8305400013923645.\n","\n","\n","Example output: Which is a species of fish? Lizard or LizardfishBanjofish\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 67%|██████▋   | 8000/11955 [1:35:02<45:53,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 8000 of 11955. Loss:0.9570227861404419.\n","\n","\n","Example output: I'm having trouble concentrating. What tips can help me focus and stay on task.Make a list of all the time-wasting activities you are trying to minimize and prioritize them. Consider the time-wasting activities ahead of time, and set aside a specific amount of time each day for yourself to focus on them.\n","2. Set a schedule of upcoming tasks and commitments that you can stick to. Take breaks when you're having trouble concentrating, but take a little bit of time to give yourself a little break too.\n","3. Make sure to practice good time management skills. Take a few deep breaths, think of a few facts or ideas for the next month, and then put together a timeline for how you want to complete them.\n","4. Find a designated workspace that is comfortable and free from distractions. This can be in a quiet space, like a room with two windows or a balcony, or even in the back of a house with a few chairs and a ceiling\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 70%|███████   | 8400/11955 [1:39:51<41:20,  1.43it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 8400 of 11955. Loss:0.6780582070350647.\n","\n","\n","Example output: What are some jobs that are around the world related to computers.Some jobs that could fit into this list include software developer, data analyst, and analyst.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 74%|███████▎  | 8800/11955 [1:44:31<36:37,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 8800 of 11955. Loss:0.6122076511383057.\n","\n","\n","Example output: What is the nearest hardware store to my location.The closest hardware store to your location is typically about halfway between you and the nearest Starbucks or Starbucks Cafe.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 77%|███████▋  | 9200/11955 [1:49:11<31:59,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 9200 of 11955. Loss:0.382173091173172.\n","\n","\n","Example output: What hit song that was used in the Bollywood movie Don actually a reject from an earlier movie.Yes, Don was a hit song that was used in the Bollywood movie Don. The song was originally a theme song from a movie called A Drama of Life. The theme song was a huge hit in the early 1980s and was one of the biggest hit song in Bollywood history.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 80%|████████  | 9600/11955 [1:53:53<27:19,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 9600 of 11955. Loss:0.5627954006195068.\n","\n","\n","Example output: What is the significance of balancing autonomy and democratic values in the governance of AI?The principle of autonomy postulates that the decision-making power to make decisions should be made by individuals only when it is feasible for them to do so. This is also known as the autonomy of individuals.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 84%|████████▎ | 10000/11955 [1:58:34<22:41,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 10000 of 11955. Loss:0.7716888189315796.\n","\n","\n","Example output: Can you review a ledger for accounting accuracy.There are many ways to review a ledger. A manual review would require a list of all entries, along with a description of what the entries are and the date they came in. An automated review could analyze the data provided by the transactions as well as the user's response. A automated response includes tracking the data as it is entered and withdrawals be monitored to ensure the entries have been properly recorded. Finally, an automated response may be used to identify any errors, errors made by the user, or errors made during the setup process.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 87%|████████▋ | 10400/11955 [2:03:19<18:04,  1.43it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 10400 of 11955. Loss:0.4592054486274719.\n","\n","\n","Example output: What is a good recipe for a vegan black bean soup.A good vegan black bean soup is made with ground black beans, sautéed onion, garlic, onion powder, and spices of your choice. You can add other spices like cumin, smoked paprika, cayenne pepper, and smoked paprika powder if you want it more nutty or sauté it in garlic butter. Enjoy.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 90%|█████████ | 10800/11955 [2:08:02<13:24,  1.43it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 10800 of 11955. Loss:0.3857034146785736.\n","\n","\n","Example output: I'm trying to memorize pi to 10 digits. Can you help me out.I'm sorry, I'm not able to help you out with this task. It is a complex matter of memorizing a number sequence with multiple numbers sequence as a sequence of numbers with multiple digits. It is difficult for me to provide you with a useful answer.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 94%|█████████▎| 11200/11955 [2:12:44<08:54,  1.41it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 11200 of 11955. Loss:0.606650710105896.\n","\n","\n","Example output: Could the \"Embedded agency\" model be applied in all industries?The concept of Embodied Agency suggests that rational agent theory should consider the agent as the primary example of the agent's current state of existence. This means that agents should be thought of as integrated into the world they exist in, rather than as distinct from it.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 97%|█████████▋| 11600/11955 [2:17:26<04:07,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 11600 of 11955. Loss:0.5445802807807922.\n","\n","\n","Example output: Does acupuncture really have a scientific basis.Yes, acupuncture has a scientific basis. Acupuncture is a form of physical therapy that focuses on the connection between the body and the environment in order to treat illnesses with a focus on acupuncture medicine. Acupuncture is based on principles of quantum theory, which posits that there are several possible ways in which the physical world could be seen, experienced, or modified by changing the behavior of a particular object. This means that many different treatments can be expected to work, with the focus being on improving patient health rather than just any physical performance.\n","\n"]},{"name":"stderr","output_type":"stream","text":["100%|██████████| 11955/11955 [2:21:39<00:00,  1.41it/s]\n"]},{"name":"stdout","output_type":"stream","text":["\n","Average Training Loss: 0.7232662604075504.\n","\n","\n","Validation loss: 0.6367488981773299.\n","\n","\n","Beginning epoch 2 of 2\n","\n"]},{"name":"stderr","output_type":"stream","text":["  3%|▎         | 400/11955 [04:38<2:14:13,  1.43it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 400 of 11955. Loss:0.4304645359516144.\n","\n","\n","Example output: What are the symptoms of a broken arm.The most common symptoms of a broken arm include numbness, tingling, tingling or tinglingness in the arms and hands; numbness or tingling on one hand; numbness or tingling on the right side; numbness or tingling in one hand; and weakness or numbness in the arms or hands.\n","\n"]},{"name":"stderr","output_type":"stream","text":["  7%|▋         | 800/11955 [09:21<2:09:42,  1.43it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 800 of 11955. Loss:0.42402124404907227.\n","\n","\n","Example output: What is the definition of intelligence?The term intelligence is often connote as an innate ability that humans possess, unlike animals or humans. However, it is widely acknowledged that humans do not have an innate ability to learn and apply new knowledge in an efficient and effective manner.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 10%|█         | 1200/11955 [14:03<2:04:54,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 1200 of 11955. Loss:0.4674263000488281.\n","\n","\n","Example output: How do you define general artificial intelligence?General Intelligence refers to the ability to effectively achieve goals across different circumstances. Artificial General Intelligence pertains to the ability to achieve goals at any given time, irrespective of the circumstances. Artificial General Intelligence is non-human inspired and relies on machine learning rather than human intellect.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 13%|█▎        | 1600/11955 [18:45<2:00:08,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 1600 of 11955. Loss:0.5717531442642212.\n","\n","\n","Example output: I have a lot of books on my shelf and I'm looking for a reliable system to organise them. What would you suggest.I would recommend starting with a book list and sticking to it. Once you’ve established the order, you can decide which books you want to keep and which you don't. It can be helpful to divide the books into “soft” and “hard” genres so that you can find the ones that you enjoy the most or that you are most able to teach effectively. Additionally, you can create categories for each book such as “useful” or “unethical” so that you can search for those that you have difficulty with. Finally, you can create sections for each book, such as “ chapters” or “ sections” so you can stay on top of the topic and get a better understanding of what you are talking about.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 17%|█▋        | 2000/11955 [23:32<1:55:24,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 2000 of 11955. Loss:0.3633972704410553.\n","\n","\n","Example output: How can I go about negotiating a higher salary.To negotiate a higher salary, it is important to do your homework, be prepared, and be confident. Before beginning the negotiation process, it is important to have an understanding of the salary range in your desired company and the compensation packages of the most qualified team members. Make sure to research salary ranges, salary packages, and other salary-related terms in case you need to make adjustments when negotiating. Additionally, be prepared to discuss salary with coworkers to help ensure an equitable and fair salary is agreed upon. Try to stay in touch with colleagues to stay up to date on current salary trends and ask for feedback and advice on how to best negotiate.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 20%|██        | 2400/11955 [28:18<1:50:48,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 2400 of 11955. Loss:0.45226219296455383.\n","\n","\n","Example output: What is the fastest way to defrost a turkey.The fastest way to defrost a turkey is to place it in the refrigerator overnight.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 23%|██▎       | 2800/11955 [32:57<1:46:13,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 2800 of 11955. Loss:0.8627551198005676.\n","\n","\n","Example output: What are some good ways to reduce anxiety.Some good ways to reduce anxiety include:\n","1. Exercise regularly.\n","2. Get enough sleep.\n","3. Get adequate sleep hygiene.\n","4. Eat a nutritious diet.\n","5. Talk about your worries or concerns.\n","6. Develop a plan to manage stress.\n","7. Prioritize tasks.\n","8. Talk to a mental health professional.\n","9. Take time for yourself.\n","10. Connect with others.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 27%|██▋       | 3200/11955 [37:41<1:41:27,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 3200 of 11955. Loss:0.3708806335926056.\n","\n","\n","Example output: What are some good dinner recipes for a family of five.Grilled Salmon and Quinoa Bowl: This vegetarian chili recipe is packed with quinoa, bell peppers, roasted vegetables, and flavorful roasted sweet potatoes.\n","\n","2. Pasta Fritters: This spicy, hearty, and filling dish is sure to be a main ingredient on any family dinner menu. \n","\n","3. Baked Pita Paws: Make your own homemade pita bread by combining flour, flour, and water in a bowl. Spread the dough onto two of your favorite pieces of greased and floured flour tortilla. Place the other piece of greased and floured flour tortilla on top and create two wedges. Fold the edges of each wrap into the center and top with a mixture of shredded cheese and vegetables.\n","\n","4. Quinoa Salad: This dish is a vegetarian or vegan-friendly, easy-to-make and nutritious meal. \n","\n","5. Black Bean\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 30%|███       | 3600/11955 [42:29<1:36:51,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 3600 of 11955. Loss:0.43039456009864807.\n","\n","\n","Example output: How should I invest my money.You should consider investing your money in a variety of different asset classes, such as stocks, bonds, mutual funds, real estate, and commodities. Consider factors like the amount of risk you are willing to take, diversification, and how often you plan to invest in each asset type. Make sure to research each option thoroughly to ensure you are making the right decisions for your situation.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 33%|███▎      | 4000/11955 [47:12<1:32:15,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 4000 of 11955. Loss:0.5732502341270447.\n","\n","\n","Example output: How can I better manage my money.One way to better manage your money is to create a budget and stick to it. You should track your expenses, prioritize needs, and minimize unnecessary spending. Additionally, try to minimize the amount of debt you have, and pay it off gradually. You may also want to create a plan for investing more in the future and cutting down on your discretionary spending. Finally, make sure to review your finances regularly and make adjustments as needed.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 37%|███▋      | 4400/11955 [51:55<1:27:37,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 4400 of 11955. Loss:0.8625234961509705.\n","\n","\n","Example output: What are common metrics used to measure the market value of a company?Common metrics used to measure the market value of a company include:\n","1. Free cash flow\n","2. Free cash flow growth\n","3. Free cash flow balance\n","4. Free cash flow ratio\n","5. Free cash flow growth rate\n","6. Free cash flow balance rate\n","7. Free cash flow growth rate in the current year\n","8. Free cash flow balance growth rate over the past 10 years\n","9. Free cash flow rate\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 40%|████      | 4800/11955 [56:38<1:23:02,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 4800 of 11955. Loss:0.37534746527671814.\n","\n","\n","Example output: What are the drawbacks and compromises associated with utilizing deep learning for minor issues?Deep learning models can be highly complex and difficult to comprehend. This makes it a challenging tool to use and understand. \n","\n","There are many different systems and techniques that can be employed to enhance the precision of deep learning models. One of the most common techniques is called 'generative adversarial networks', orGANs for short. \n","\n","Generative Adversarial Networks (GANs) are a type of deep learning model that consists of two distinct neural networks: a generator and a discriminator. The generator network generates new data samples, while the discriminator network evaluates the authenticity of the generated samples.\n","\n","There are two different approaches to utilizing deep learning for minor issues:\n","\n","1. Using the generator to identify and comprehend the underlying issues that the model is trained on\n","\n","2. Utilizing the discriminator to identify the potential issues that the model is not aware of\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 43%|████▎     | 5200/11955 [1:01:27<1:18:20,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 5200 of 11955. Loss:0.3612576723098755.\n","\n","\n","Example output: Who starred as Rocky Balboa Rocky Balboa.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 47%|████▋     | 5600/11955 [1:06:06<1:13:41,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 5600 of 11955. Loss:0.4218217730522156.\n","\n","\n","Example output: What's the best way to cook frozen french fries.The best way to cook frozen french fries is to thaw them in a bowl with a little oil and salt, then heat them in a skillet over medium heat, flipping them occasionally. You may also add diced vegetables like peppers and onions to the pan to taste, or add some butter or olive oil to the pan and swirl the mixture around. Finally, add a splash of mustard or mustard vinaigrette and a dash of garlic oil. Enjoy.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 50%|█████     | 6000/11955 [1:10:49<1:09:01,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 6000 of 11955. Loss:0.40789592266082764.\n","\n","\n","Example output: What are the steps needed to start my own business.Decide what type of business you would like to start and what business you would like to succeed in.\n","2. Register your business with the appropriate authorities and obtain any necessary licenses and permits.\n","3. Obtain financing to cover startup costs such as equipment, supplies, materials, and any other required startup costs.\n","4. Register your business with the relevant tax and other regulations.\n","5. Obtain the necessary insurance coverage to cover liability for damage to business assets or business processes during operations.\n","6. Purchase a business location if one is not already available in your area.\n","7. Launch your business and begin operations.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 54%|█████▎    | 6400/11955 [1:15:35<1:04:29,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 6400 of 11955. Loss:0.49317994713783264.\n","\n","\n","Example output: what is the most important thing large language models can do?The most important thing large language models can do is to provide you with a comprehensive representation of the language, which allows you to identify patterns and make accurate predictions. These models don’t have to be able to generate the exact same language, just a representation of it.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 57%|█████▋    | 6800/11955 [1:20:16<59:46,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 6800 of 11955. Loss:0.45603877305984497.\n","\n","\n","Example output: Is it possible for deep learning models to apply transfer learning to tasks that are not related?Deep learning models can be trained to recognize patterns in data, such as making predictions or making predictions based on past data. Transfer learning is a technique where a pre-trained model is used as a foundation for a new model that is trained on a different dataset. In transfer learning, the pre-trained model is used as a feature extractor or a feature extractor model, and the new model is trained using a different dataset. Transfer learning is a significant challenge in deep learning, as it requires the transfer of knowledge from the pre-trained model to the new model, which is a significant challenge. Transfer learning can be done through a combination of methods, including transfer learning with transfer learning, transfer learning with generative modeling, and transfer learning with transfer learning. Transfer learning with transfer learning is a type of generative model, where a pre-trained model is used as a feature extractor or a\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 60%|██████    | 7200/11955 [1:25:05<55:09,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 7200 of 11955. Loss:0.4477548897266388.\n","\n","\n","Example output: What are the top ten books to read right now.Pebble Beach \n","\t\t\t2  Pride and Prejudice by Jane Austen\n","\t\t\t3  Pride & Prejudice by Jane Austen\n","\t\t\t4  Pride & Prejudice by Jane Austen\n","\t\t\t5  Pride & Prejudice by Jane Austen\n","\t\t\t6  Pride and Prejudice by Jane Austen\n","\t\t\t7  Pride and Prejudice by Jane Austen\n","\t\t\t8  Pride and Prejudice by Jane Austen\n","\t\t\t9  Pride and Prejudice by Jane Austen\n","\t\t\t10 Pride and Prejudice by Jane Austen.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 64%|██████▎   | 7600/11955 [1:29:51<50:30,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 7600 of 11955. Loss:0.4525115489959717.\n","\n","\n","Example output: How do I pass the ACT test.To pass the ACT test, you will need to familiarize yourself with the material and create a study plan. Once you have chosen a study material, practice test practice is a great way to review what you have learned. It is also helpful to create a study plan that allows you to review the material in different environments to test your knowledge in different situations. Additionally, it can be helpful to review material regularly and get used to new concepts. Finally, be sure to review key words and concepts to make sure that you can easily recall them at the test.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 67%|██████▋   | 8000/11955 [1:34:36<45:49,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 8000 of 11955. Loss:0.34699341654777527.\n","\n","\n","Example output: What is the capital of Iceland.The capital of Iceland is Reykjavik.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 70%|███████   | 8400/11955 [1:39:14<41:11,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 8400 of 11955. Loss:0.584662675857544.\n","\n","\n","Example output: Could you elaborate on the meaning of Stoicism?The philosophy of Stoicism, which originated in ancient Greece and was later developed in Rome, emphasizes the importance of self-discipline, reason, and virtue in attaining a fulfilling life. According to Stoics, it is essential to concentrate on what we can control and acknowledge the unchangeable, rather than dwelling on what we cannot. This approach can help us to be more accepting of mistakes, find joy in our own stories, and find peace in the face of adversity.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 74%|███████▎  | 8800/11955 [1:43:58<36:33,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 8800 of 11955. Loss:0.47312238812446594.\n","\n","\n","Example output: What is a good, affordable place to stay when visiting Washington D.C.A great place to stay for an affordable price is to rent a cabin or camping site near the Washington Monument. There are plenty of affordable campgrounds located just near the monument, and they have great amenities, with plenty of dining options and plenty of green space for exercise.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 77%|███████▋  | 9200/11955 [1:48:39<31:56,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 9200 of 11955. Loss:0.35052648186683655.\n","\n","\n","Example output: What are some common applications of deep learning?Deep learning has a wide range of applications in various fields, including:\n","\n","1. Image and speech recognition: Deep learning algorithms can be used to analyze images and speech.\n","\n","2. Natural language processing: Deep learning algorithms can be used to understand and process natural language.\n","\n","3. Healthcare: Deep learning algorithms can be used to analyze medical images and diagnose diseases.\n","\n","4. Finance: Deep learning algorithms can be used to analyze financial data and make informed investment decisions.\n","\n","5. Finance: Deep learning algorithms can be used to analyze financial data and make informed investment decisions.\n","\n","6. Finance: Deep learning algorithms can\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 80%|████████  | 9600/11955 [1:53:25<27:17,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 9600 of 11955. Loss:0.37652677297592163.\n","\n","\n","Example output: How would you define objective function?In the realm of mathematical optimization, the concept of a function that is a real-valued function of the real input is called a real-valued function of the real output.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 84%|████████▎ | 10000/11955 [1:58:06<22:41,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 10000 of 11955. Loss:0.37584447860717773.\n","\n","\n","Example output: What kinds of jobs would I be good at.Some potential job opportunities that you could consider include software engineer, data scientist, computer programmer, digital marketer, web developer, system administrator, network engineer, and project manager.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 87%|████████▋ | 10400/11955 [2:02:46<18:03,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 10400 of 11955. Loss:0.41572290658950806.\n","\n","\n","Example output: What is the relationship between mind and body?The relationship between the mind and body has been a topic of discussion for centuries. Dualism is one perspective that proposes the mind and body are separate entities, while monism suggests they are distinct aspects of the same entity. Idealism, materialism, and functionalism also play role in this discussion. Idealism proposes that the mind and body are objective, while materialism suggests they are subjective. functionalism suggests that the mind is a product of the body and vice versa. Understanding the relationship between the mind and body is essential for understanding the mind and body's relationship to each other.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 90%|█████████ | 10800/11955 [2:07:32<13:24,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 10800 of 11955. Loss:0.41179633140563965.\n","\n","\n","Example output: How does deep learning impact society as a whole?Deep learning has the potential to transform many aspects of our lives, from healthcare to finance, from self-driving cars to autonomous vehicles, from data analytics to predictive analytics. Some of the applications are already being launched, with more being developed each year. For instance, deep learning algorithms have been shown to be very successful in image and speech recognition, natural language processing, and language translation.\n","\n","Overall, deep learning has the potential to transform many aspects of our lives. It is critical to have a well-defined and comprehensive understanding of the potential impact of deep learning on our society.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 94%|█████████▎| 11200/11955 [2:12:17<08:45,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 11200 of 11955. Loss:0.7317464351654053.\n","\n","\n","Example output: Do we need to prepare for rain tomorrow?I am not capable of predicting the future. My scope of knowledge is limited to a specific task, and I am not capable of understanding the broader societal, structural, and economic factors that contribute to this task.\n","\n"]},{"name":"stderr","output_type":"stream","text":[" 97%|█████████▋| 11600/11955 [2:16:57<04:07,  1.44it/s]"]},{"name":"stdout","output_type":"stream","text":["\n","Batch 11600 of 11955. Loss:0.3916742205619812.\n","\n","\n","Example output: What is the nature of good and evil?The nature of good and evil is a complex philosophical and ethical question that has been debated for centuries. In general, good and evil can be described as a continuous and balanced combination of actions and consequences. They can also be described as a continuous and balanced combination of consequences and desires. In other words, good and evil are a combination of good and evil emotions, thoughts, beliefs, and behaviors. However, it is important to note that good and evil can also be influenced by external factors such as social and cultural factors, personal desires, and personal circumstances. Ultimately, it is up to each individual to decide for themselves the nature of good and evil and the consequences of their actions.\n","\n"]},{"name":"stderr","output_type":"stream","text":["100%|██████████| 11955/11955 [2:21:12<00:00,  1.41it/s]\n"]},{"name":"stdout","output_type":"stream","text":["\n","Average Training Loss: 0.48824751933889377.\n","\n","\n","Validation loss: 0.5940002547093493.\n","\n","Training complete!\n"]},{"data":{"text/plain":["('/content/drive/MyDrive/Colab Notebooks/Aira-1.5B/tokenizer_config.json',\n"," '/content/drive/MyDrive/Colab Notebooks/Aira-1.5B/special_tokens_map.json',\n"," '/content/drive/MyDrive/Colab Notebooks/Aira-1.5B/vocab.json',\n"," '/content/drive/MyDrive/Colab Notebooks/Aira-1.5B/merges.txt',\n"," '/content/drive/MyDrive/Colab Notebooks/Aira-1.5B/added_tokens.json')"]},"execution_count":11,"metadata":{},"output_type":"execute_result"}],"source":["from codecarbon import EmissionsTracker\n","import tqdm\n","\n","output_dir = f'your_path_to_output_dir'\n","\n","tracker = EmissionsTracker(\n","    project_name=\"Aira_emissions\",\n","    log_level=\"critical\",\n","    output_dir=output_dir,\n","    output_file=\"Aira_emissions.csv\",\n",")\n","\n","training_stats = []\n","\n","tracker.start()\n","for epoch_i in range(0, epochs):\n","\n","    print(f'\\nBeginning epoch {epoch_i + 1} of {epochs}\\n')\n","\n","    total_train_loss = 0\n","\n","    model.train()\n","\n","    for step, batch in enumerate(tqdm.tqdm(train_dataloader)):\n","\n","        b_input_ids = batch[0].to(device)\n","        b_labels = batch[0].to(device)\n","        b_masks = batch[1].to(device)\n","\n","        model.zero_grad()\n","\n","        outputs = model(b_input_ids,\n","                        labels=b_labels,\n","                        attention_mask = b_masks,\n","                        token_type_ids=None)\n","\n","        loss = outputs[0]\n","\n","        batch_loss = loss.item()\n","        total_train_loss += batch_loss\n","\n","        if step % sample_every == 0 and not step == 0:\n","\n","            print(f'\\nBatch {step} of {len(train_dataloader)}. Loss:{batch_loss}.\\n')\n","\n","            model.eval()\n","\n","            inputs = tokenizer(tokenizer.bos_token + df.prompt.sample().iloc[0] + tokenizer.eos_token, return_tensors=\"pt\").to(device)\n","\n","            sample_outputs = model.generate(**inputs,\n","                                bos_token_id=tokenizer.bos_token_id,\n","                                pad_token_id=tokenizer.pad_token_id,\n","                                eos_token_id=tokenizer.eos_token_id,\n","                                do_sample=True,\n","                                top_k=50,\n","                                max_length = 200,\n","                                top_p=0.95,\n","                                num_return_sequences=1)\n","\n","            for i, sample_output in enumerate(sample_outputs):\n","                  print(f'\\nExample output: {tokenizer.decode(sample_output, skip_special_tokens=True)}\\n')\n","\n","            model.train()\n","\n","        loss.backward()\n","\n","        optimizer.step()\n","\n","        scheduler.step()\n","\n","    avg_train_loss = total_train_loss / len(train_dataloader)\n","\n","\n","    print(f'\\nAverage Training Loss: {avg_train_loss}.\\n')\n","\n","    model.eval()\n","\n","    total_eval_loss = 0\n","    nb_eval_steps = 0\n","\n","    for batch in validation_dataloader:\n","\n","        b_input_ids = batch[0].to(device)\n","        b_labels = batch[0].to(device)\n","        b_masks = batch[1].to(device)\n","\n","        with torch.no_grad():\n","\n","            outputs  = model(b_input_ids,\n","                             attention_mask = b_masks,\n","                             labels=b_labels)\n","\n","            loss = outputs[0]\n","\n","        batch_loss = loss.item()\n","        total_eval_loss += batch_loss\n","\n","    avg_val_loss = total_eval_loss / len(validation_dataloader)\n","\n","\n","    print(f'\\nValidation loss: {avg_val_loss}.\\n')\n","\n","    training_stats.append(\n","        {\n","            'epoch': epoch_i + 1,\n","            'Training Loss': avg_train_loss,\n","            'Valid. Loss': avg_val_loss,\n","        }\n","    )\n","\n","tracker.stop()\n","print(\"Training complete!\")\n","\n","df_stats = pd.DataFrame(data=training_stats)\n","df_stats = df_stats.set_index('epoch')\n","df_stats.to_parquet(f\"{output_dir}/training_stats.parquet\", compression=\"gzip\")\n","\n","model_to_save = model.module if hasattr(model, 'module') else model\n","model_to_save.save_pretrained(output_dir)\n","tokenizer.save_pretrained(output_dir)"]},{"cell_type":"markdown","metadata":{"id":"h48iOsqie8rx"},"source":["13. Check the training stats and plot the learning curves."]},{"cell_type":"code","execution_count":4,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":361},"executionInfo":{"elapsed":2519,"status":"ok","timestamp":1687474618432,"user":{"displayName":"Nicholas Corrêa","userId":"09736120585766268588"},"user_tz":-120},"id":"J1-hAY9Av4gT","outputId":"b32c6ae9-0f87-4411-8f38-a89a07f9ec7e"},"outputs":[{"data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAABBsAAAI/CAYAAAA2r9HeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAACcsUlEQVR4nOzdd3iUVf7+8feUJKQXICGUEAgQaqgKCFItKChFQF11bT9Rse267urqytp1v+vqKvaCKCu7gjQbZUFqaKGE0CEhjQAJkEoCKTPz+yNmlpgEkjBhZpL7dV1c6jznOc9nQg4yd04x2Gw2GyIiIiIiIiIiDmJ0dgEiIiIiIiIi0rgobBARERERERERh1LYICIiIiIiIiIOpbBBRERERERERBxKYYOIiIiIiIiIOJTCBhERERERERFxKIUNIiIiIiIiIuJQChtERERERERExKEUNoiIiIiIiIiIQylsEBGRJmfLli1ER0cTHR3t8L4XLlxIdHQ0o0aNcnjf4hh33XUX0dHRzJw5s07XLrXvy2HUqFFER0ezcOFCpzxfRESkgtnZBYiISON0KR/kX3/9dSZNmuTAaqSudu3axaxZs9i+fTu5ubkEBgYSHh7OkCFDuOGGG+jatWu9+j1+/DijRo3CarXypz/9ifvvv79W9y1evJinn34aKA90evToUa/nu6uFCxeSkZHBlVdeycCBA51djsM988wzLFq0iDZt2vDzzz87uxwREXEAhQ0iItIgWrRoUe3rRUVFFBUVXbBNs2bNGqwuAG9vbzp06NAgffv7+9OhQwfCwsIapP/L4dtvv+X555/HarUC5V+voqIidu/eze7du9mxYwdz5sypV9/h4eFcddVVbNiwgYULF9Y6bFiwYAEA3bp1a9CgITw8nA4dOhAcHNxgz6iPRYsWsXXrVh599NELhg3t2rXD09MTf3//y1idiIhIVQobRESkQcTGxlb7+syZM3nvvfcu2KahxcTEsGzZsgbp+9prr+Xaa69tkL4vh+zsbF566SWsVivdunXjlVdeoWfPngCkp6fz888/c+TIkUt6xuTJk9mwYQOJiYns2rWL3r17X7B9eno6cXFxANxyyy2X9OyL+b//+78G7b+hffnll84uQUREBFDYICIiIufZtm0bxcXFAPz973+nc+fO9mvt2rXj7rvvvuRnjB49mqCgIHJzc1mwYMFFw4aFCxdis9nw9PTkpptuuuTni4iISMNT2CAiIi6lYq+Hr776ik6dOvHJJ5+wZs0aTpw4wblz5zh48CAAZ8+eZdWqVaxbt46DBw+SmZnJmTNnCAoKIiYmhltvvZXhw4dX+4wtW7bw29/+FsDeX4WFCxfy5z//2b52fM+ePXz66af2vQvCwsK45pprmD59OoGBgVX6/vX956uY1XHllVcyZ84cNm3axBdffEFCQgKFhYW0bduWsWPH8sADD+Dl5VXj12jlypV89dVX7Nu3D4vFQrt27bjpppu45557+Oijjyo9o65MJpP93xtqKYinpyfjx4/nyy+/5Mcff+TZZ5+tcemM1Wpl8eLFQPmskaCgIAAOHTrE8uXLiYuL49ixY2RlZWE2m4mIiGD48OHcfffdhISE1Lm2u+66y75c4bHHHqty3WKxMHfuXBYuXEhycjKenp5ER0dzxx13MGbMmAv2nZ6eztKlS9myZQtHjx4lMzMTg8Fg3wvj3nvvpXXr1pXuqfh+qvDee+/ZZwZVWLVqFW3btgXKN4jMyMiocd8Ti8XCokWL+O677zh48CCFhYUEBwfTt29f7rjjjhqXaJz/dXn00UeZP38+8+fPJykpCZvNRpcuXfjNb37D+PHjL/g1aAgnT55k1qxZrFu3joyMDADatGnD8OHDue+++2pcrpWXl8fs2bNZs2YNqamplJSUEBgYSEhICH379uWGG25g8ODBle45d+4cX3/9NStWrODIkSMUFRXh7+9PSEgIvXr1YtSoUVx//fUN/p5FRNyBwgYREXFJaWlpPPnkk5w6dQovLy/M5sr/y1q6dKn9Q5jBYMDPzw+z2czJkydZtWoVq1at4r777rNvKlgf33//PX/+858pLS3F398fi8XC0aNHmT17NrGxsXzzzTf4+vrWq+/PPvuMN998Eyjf56G0tJQjR44wc+ZMtm7dyhdffFHpg3+Fv/3tb8yaNcv+3wEBASQlJfHmm2+ydu1a+vfvX783+4vBgwcTEhJCdnY2X331FY8++ugl9VeTyZMn8+WXX3LmzBmWL19e44fUTZs2cezYMaDyEoqHHnrI/sHSy8sLb29v8vLy2L9/P/v372fRokXMnj2bjh07OqzmkpISHn74YTZs2ACA0WjEw8ODuLg4tm7dygMPPHDB+5999lm2bt0KgIeHB76+vuTn55OUlERSUhKLFi3io48+YsCAAfZ7mjVrRosWLcjLy6O0tBQfHx98fHwq9Vvd90l1CgoKmD59ur0Gk8mEr68vJ0+eZPny5SxfvvyiY8ZisfDII4+watUqzGYzzZo1o7CwkPj4eOLj40lNTeXxxx+vVT2OsHXrVh555BHy8/MB7F+bxMREEhMT+fbbb/nggw8qfU0BTpw4we23327/3jIajfj7+5OTk8OpU6c4dOgQycnJlcKGM2fOcMcdd3DgwAGg/M8df39/CgoKyMnJISkpibi4OIUNIiK/0NGXIiLikl577TX8/f2ZPXs28fHx7Nixo9I+CwEBAdx3333MnTuXnTt3sm3bNuLj41m/fj2PPfYYHh4ezJo1i1WrVtXr+dnZ2Tz77LNMmDCBNWvWsG3bNnbs2MGMGTPw8PDg8OHDfPbZZ/Xq+8CBA/zjH/9g2rRpbNy4kbi4OLZt28YjjzwClM+8WLRoUZX7fvzxR3vQMG7cONatW0dcXBw7duzg5ZdfJiEhgX//+9/1qqmCj4+P/cPm+++/z3fffXdJ/dWkS5cuxMTEAP/b/LE6FdfatGlT6YPfFVdcwRtvvMHq1atJSEhgy5YtJCQkMHv2bGJiYsjMzOSpp55yaM3/+Mc/2LBhAwaDgd/97nfExcURFxdHbGwst99+O59++in79++v8f6uXbsyY8YMli9fbq959+7dzJ8/n6uvvpqCggJ+//vfc+7cOfs9N954I7GxsfTt2xeA++67j9jY2Eq/wsPDa1X/c889x9atW/Hw8OAvf/kL27dvJy4ujvXr19uDnFmzZl3we2ju3Lls3bqVN954g+3bt7N9+3bWrl3LyJEjAfjwww9JSUmpVT2X6vjx4/agoVOnTvY/C3bu3MnXX39Nhw4dyMvL45FHHiEzM7PSvTNnzuTYsWO0adOG2bNns2fPHrZu3cru3bv5+eefeeGFF6os7/nqq684cOAAQUFBzJw5k4SEBOLi4ti9ezfr1q3jb3/7G0OGDLks711ExB0obBAREZdkNBqZPXs2gwcPxmgs/9/V+SdIXHPNNTz99NP0798fb29v++uhoaE8+uij/P73vweo96kJZ8+eZezYsbzyyiv2D3Pe3t7ccccd3HnnnUD5h//6yM/PZ/r06Tz55JP2qf5+fn48/vjjXHfdddX2bbPZeOeddwAYMmQIb775pn2Zg5eXF1OnTuWFF14gLy+vXjVVyMjIsIcoVquVZ5555oJhwKWYPHkyUP7T6fT09CrX8/LyWLlyJQCTJk2yfx9A+QyPiRMnVlp24OnpyeDBg5k9ezYtWrRg7969bNu2zSG1ZmZm8q9//QuAhx9+mIcffhg/Pz8AmjdvzgsvvMC4ceMoKCiosY/nnnuOO+64g8jISPt7MZvNxMTE8PHHHxMdHU1WVhbLly93SM3n27Vrl73f559/nrvuuss+blq2bMlrr71m/4n8O++8Y9+349fy8vJ47733mDhxon3pS6tWrXj33XcJDQ3FarWydOlSh9dfnY8++oj8/HwCAwOZPXt2pVk9AwYMYPbs2fj5+ZGbm8vHH39c6d6dO3cC8OSTTzJ48GD77BCTyUSbNm24/fbbq4RVFffcd999XHfddXh6egLlf1aFhYUxYcIEXn755QZ7vyIi7kZhg4iIuKTx48fTqlWret8/YsQIAOLj47FYLPXq4+GHH6729dGjRwOQmprK2bNn69yvp6cn99133wX7/vVeEvv37yc1NRWABx98EIPBUOXeX3/4rqu8vDzuvvtuDh8+zO23384777yDwWDgueeeqzG0+frrr4mOjq7X1PGxY8fi7e2NzWardibHDz/8QHFxMUajkYkTJ9a6X19fX6644goAduzYUee6qrN8+XLKyspo1qxZjcd1XsqSE5PJxNVXXw3A9u3b691PTX766SegPBiYMmVKtW2eeOIJAHJycmo8KaZfv34MGjSoyuuenp4MHToUqPq92xBsNpt9ptNtt91Gy5Ytq7Rp1aoVt912G1A1vAsICADK93uorfrcIyLSlGnPBhERcUn9+vW7aJtTp04xd+5cYmNjSUlJoaCgoEqwcPbsWfLy8uq8WWBQUBDt27ev9lpoaKj93/Pz8yvNrKiNzp0717jXQ0Xfv56hsHfvXqB8rX/FlPpfMxgMXHHFFSxZsqRO9VR45ZVXSE9Pp0+fPjz//POYTCYsFgt//OMfeeWVVygqKuLBBx+sdE/F9PRu3brV+Xl+fn5cf/31LF68mMWLF/Poo49Wmr1QMaNi8ODBtGnTpsr9q1evZsmSJezevZvTp09XG/ycOHGiznVVZ8+ePQD07NnTPqPh1zp06EBYWFiVKfvn27ZtG99++y3x8fFkZmZSVFRUpc2F7q+vivoHDhxY6Wt8vqioKHv9e/bsYdSoUVXaXOjkkJq+dxvC0aNHyc3NBaiyieP5hgwZwmeffUZubi7p6em0a9cOKA8jd+7cyT/+8Q+OHDnCtddeS79+/Wr8va2454cffuBf//oX2dnZ3HjjjfTr169eG5GKiDQFChtERMQlNW/e/ILXd+7cybRp0+wbw0H5fgPe3t4YDAYsFgs5OTkA9Zp9cKGNH8/fkK+0tLRB+i4rK6v0esV7CQoKsk/frk59T5A4efKk/aff06dPt9cxduxYSktL+fOf/8xbb71FYWEhTz75pP2+uLg4APua/bqaPHkyixcvJiMjg02bNtnXvB84cMAesFQst6hgtVr54x//yA8//GB/zWw2ExgYiIeHB1C+GWJxcXG9fu+rc/r0aeDiX99WrVrVGBb8/e9/r7TPh8lkqlRzUVGR/Zej1bX+iva/dqHv3YpNXH/9vdsQzq/vQu/p/GvZ2dn2sOH+++/nwIEDLF26lHnz5jFv3jwMBgOdO3dm6NChTJkypcrmojfddBMJCQn861//4scff7TPlmjfvj1DhgzhlltuoWfPno58myIibk3LKERExCXV9NNXKP8w84c//IH8/Hy6devGJ598wvbt29m5cycbN24kNjaWefPm2dvbbLbLUbJb27dvn/1D4q9PtJgwYQKvvPIKBoOBjz/+mFdeeQWbzcaRI0fYuXMngYGBXHPNNfV67hVXXEFkZCRQfsxjhYp/DwoKqtL3t99+yw8//IDJZOKRRx5hxYoV7N69m61bt9o3TaxY1uEqv/exsbH2oOE3v/kN33//fZWa7777bidX2XR4eHjwz3/+kyVLlvDII48waNAgvL29OXToELNmzWLcuHGVTn2p8Nxzz7Fs2TKefPJJhg0bRkBAAKmpqcydO5dbbrmFV1991QnvRkTENWlmg4iIuJ34+HgyMjIwmUx8/PHH1f5ks7Gtqw4ODgYgNzeXkpKSGmc31HcKfmFh4QWv33LLLZSVlfHXv/6VOXPmUFhYSH5+PjabjbvvvrveR4BW9P2Pf/yD//73v/ZlKRWnYNx0001V3mvFT5QnT55c4zGLp06dqnc91amYaXOxr29N1ytqHjp0KH/961+rbePoms/XvHlzkpOTL7qspOL6xWYWOdv59WVmZtZ4xOn5vx/VLXfo2rUrXbt2BcpDzLi4ON5//33i4uL4v//7P6666ir79Qrt27fnwQcf5MEHH8RqtZKQkMCnn37KypUr+eqrrxg0aJB97xURkaZMMxtERMTtHD9+HCj/8FDTFOpNmzZdzpIaXI8ePYDyZRsVu+L/ms1mq/fpCxXTywE2b95cbZtbb72V559/HiifebBy5Uo6dOjA//t//69ez6wwYcIETCYTxcXFfP/99/z888/2ZSO/XkIB//tA3L1792r7KywsZNeuXZdU069VTI/fs2dPjcFMSkpKjR/mL1azzWar8esO2DcEre9MjYr6t2zZgtVqrbZNUlKS/cN5r1696vWcy6Vt27YEBQUBFx7rGzduBMpnyJz/PV4ds9nM4MGD+fjjj/H09MRms9nvr4nRaKRPnz68++679s1ZL3aPiEhTobBBRETcjr+/P1D+k+Dqfhp84sSJeh956aq6detm37Dyk08+qfZD55IlS8jIyKhX/z179iQiIgIo31ug4sP+r91xxx3ccMMN9v/u2rUrXl5e9XpmhdDQUIYNGwaUhxgVSyh69OhR5afKgH0TvwMHDlTb3wcffHDRmRp1df3112MymTh37ly10+sB3n///Rrvv1jN//73v6s9/vPX95+/R0ldjB07Fij/Sf/8+fOrbfPuu+8C5bNorrrqqno953IxGAz278Nvvvmm2plMmZmZfPPNNwCMGzeu0rWSkpIa+/b09LTvWXL+cq4L3WMymex7b1R3UoyISFOksEFERNxO//798fHxwWaz8bvf/Y7k5GQALBYL69ev56677nJyhY5nMBh47LHHANiwYQNPP/20/afQxcXFzJ8/n7/+9a8EBgbWu/8ZM2ZgMplISUlhypQpLF++nOLiYqD8a7tjxw4ef/xxli5dav9AtXTpUt5+++1Lfn8VMxj27NnDunXrgPLlFdWpOCJy/vz5fPPNN/YPgSdPnuS1117js88+s//U21HCwsL4zW9+A5SHGR9//DFnzpwByjcefOmll/juu+/sQVhNNa9bt47333/fvglkfn4+H330Ea+88soFa+7cubP9/voslYmJibHvY/Hyyy/zr3/9y7555smTJ/nLX/5iP0ryiSeeuOQAqb6sVivZ2dkX/FXxdX/ooYcICAggNzeXe++9t9Ixp9u3b+fee+8lPz+foKAgpk2bVuk5I0eO5B//+Afx8fGVQoTU1FSeeuopzp49i9FotB/nCTBlyhReeeUVtmzZUmkTz8zMTF5++WX70bTDhw9vkK+NiIi70Z4NIiLidvz9/fnTn/7ECy+8QFxcHGPGjMHHxweLxUJxcTHBwcG8/vrrPPzww84u1aFuuukmdu/ezZdffsmSJUv47rvvCAgIoKioiNLSUgYNGkTv3r3t08Dr6uqrr+att97iueeeIz09nccffxyz2Yyfnx+FhYX2kzdat27Na6+9xrp165g1axYfffQRLVu25M4776z3exsxYgQtWrTg1KlTWK1WvLy8uOmmm6pte99997F8+XKOHDnCjBkzeOGFF/Dz86OgoACbzcatt95KSUkJixYtqnc91fnjH/9IUlISGzdu5K233uKdd97Bz8/PvnfFAw88wK5du9i6dWuVeydMmMDixYvZtm0b7777LjNnziQgIICCggKsVisjRoygW7dufPjhh9U+e+LEiXzxxRekpqYyYsQIQkJC7IHA3LlzadWq1UXrf/XVV8nJyWHr1q28/PLLvP766/j6+trrh/Kv7e23334JX6VLc/z48QseZQkwevRoPvjgA1q1asX777/P9OnTOXz4MLfffjs+Pj4A9jAgICCA999/v8pyq1OnTvHJJ5/wySefYDQa8ff359y5c/ZwzWAw8PTTT9OpUyf7PQUFBcyZM4c5c+ZgMBjw9/enrKysUvBwzz332IMlEZGmTmGDiIi4pdtvv53WrVvz2WefsWfPHiwWC2FhYQwfPpwHHnigXkdSuoNnn32WK664gq+++op9+/ZRUlJCx44dGT9+PHfffTdvvPEGUP4hqz7GjBlDv379mDt3LuvWrSM1NZXCwkKCgoLo0aMH1157LTfffDOenp4MHDiQlJQUfv75Z1599VWaN29eaYlFXZjNZiZMmGA/seHaa6+t8T0EBATwn//8h/fff5+VK1eSlZWFyWTiyiuv5NZbb2Xs2LE888wz9arjQry8vPj000+ZO3cuCxcuJDk5GZvNxoABA+zLS2qaVePh4cGsWbP45JNP+OGHH8jIyMBmsxETE8OECRO49dZbL7gMIzIykq+++oqPP/6YhIQEcnNz7aeH1PaoSX9/f2bPns2iRYtYsmQJBw8epKioiBYtWtCvXz/uuOMOBg4cWPcvjBNdeeWV/PTTT3zxxResXbuWjIwMDAYDUVFRDB8+nPvuu4+WLVtWuW/WrFls2bKF7du3c/z4cftyrPbt29O/f3/uuOOOKsdYvvXWW2zYsIFt27Zx9OhRTp06RVlZGW3atKF3795MnTr1okGJiEhTYrC5yplQIiIicsluu+02du7cyeOPP84jjzzi7HJERESkidKeDSIiIo3E1q1b7SdVaCq3iIiIOJPCBhERETfy4osvsnDhQk6ePGlfZ5+fn89//vMfpk+fDsCgQYOIiYlxZpkiIiLSxGkZhYiIiBsZP368/fhET09PvL29K23w16lTJ2bNmlVlQzwRERGRy0lhg4iIiBtZtWoVK1euJCEhgVOnTnHmzBn8/Pzo1KkT1157Lbfeeive3t7OLlNERESaOIUNIiIiIiIiIuJQ2rNBRERERERERBxKYYOIiIiIiIiIOJTZ2QXIhdlsNqzW2q10MRoNtW4rIpeHxqWI69G4FHEtGpMirsVoNGAwGC65H4UNLs5qtZGdXXjRdmazkeBgX/Lziygrs16GykTkYjQuRVyPxqWIa9GYFHE9ISG+mEyXHjZoGYWIiIiIiIiIOJTCBhERERERERFxKIUNIiIiIiIiIuJQChtERERERERExKEUNoiIiIiIiIiIQylsEBERERERERGHUtggIiIiIiIiIg6lsEFEREREREREHEphg4iIiIiIiIg4lNnZBYiIiIiIiDQ0m82GxVKGzWZzdikil4XRaMRkct5HfoUNIiIiIiLSaJWVlVJQkEtJyTlsNquzyxG5rMxmT3x9A/D29r38z77sTxQREREREbkMSkqKycnJwmg04uvrj4eHF0ajETA4uzSRBmbDYrFQVHSGvLxTAJc9cFDYICIiIiIijdKZM7mYTGZCQsJ+CRlEmg4PD/Dy8iYn5ySFhfmXPWzQiBMRERERkUbHYrFQUnIOX19/BQ3SZBkMBnx8fCkrK8FiKbusz9bMhkbAarWxPyWb0uQcPAw2oloHYjRqapiIiIiINF1WqwUAs9nDyZWIOFfFJpFWqxWT6fI9V2GDm9t+MIu5Kw+TU1Bsfy3Y34vfXNOZ/tGhTqxMRERERMQV6Idw0tQ5ZwxoPpEb234wi/cX7akUNADkFBTz/qI9bD+Y5aTKREREREREpClT2OCmrFYbc1cevmCbf688jNWqc4RFRERERETk8lLY4KYOpedWmdHwa9kFxRxKz708BYmIiIiIiIj8QmGDm8otvHDQUNd2IiIiIiIiruDVV19g6NAB/PTT9w7r89FHpzF06AB27NjmsD7lwrRBpJsK8vVyaDsREREREWmahg4dUK/75s//jvDw1g6uRhoLhQ1uqku7IIL9vS64lCLE34su7YIuX1EiIiIiIuJ2evXqXeW10tJSDhzYB0DXrt3x8Kh6hKinp2eD1NO8eQsiItrj6+vnsD7DwloREdGeZs2aOaxPuTCDzWbTDoIuzGKxkp1dWO21itMoavLIxJ46/lLEicxmI8HBvuTkFFJWZnV2OSKCxqWIq2nIMVlaWsLp08dp3jwcD4+G+VDcmB0/fowpU24GNIPB3dV1LISE+GIyXfqOC9qzwY31jw7lkYk9CfavulQi0M+T3p1aOKEqERERERERaeq0jMLN9Y8OpW/nliQdy6PUZsBaVsYn3+0l70wJ63cdY2S/ts4uUURERESkSbFabRxKzyW3sJgg3/KlzUajwdllOcz5sx42bNjG2rWrmT//3yQlJVJQkM8XX3xN587RnD59ijVrfmbTpg2kpaVy6tQpzGYz7du3Z9So67jllqnVLsV49dUXWLr0B5599q/ceONN9td/+ul7XnvtRfr06cfMmR+zZMkClixZSFpaKp6eXvTp05cHHphOx45RVfp89NFpxMfv4N13P6Jfv//tUfH55x/zxRefcsMN43j66b/w73/PYdmyHzl+/Bg+Pr4MHDiIadMeISysVbVfi5Mns/jss4/YvHkjBQX5hIaGMXr0dfz2t/fx5puvV/s+mgqFDY2A0WigW2SIfQra+KEd+fq/h1iyIZlBPVrh7aXfZhERERGRy2H7wSzmrjxcaW+1YH8vfnNN50a5xPnrr7/kww9nEhQUTNu2bcnKyrRf+/77xXz22Ud4enrRvHkLoqKiyMvL49Chg+zfv49161bz7rsfVbsfxMW88spfWb78J8LDWxMR0Z7U1FTWr1/Lzp3b+eyzObRt265O/ZWVlfGHPzzG9u1xtGsXQdu27UhLS2X58qXs3LmD2bPnEhAQWOmetLRUHnnkAXJysjGbzXTsGEVxcTFffvk527ZtbfJLT/QptBEa3qc1K7elk5lzluVb05hwdUdnlyQiIiIi0ujVtKdaTkEx7y/a0yj3VPvss4948smnmTDhFoxGI1arFYvFAkDfvgN4++336du3P2bz/z56ZmVl8vbbf2f9+jX85z//4q677q3TM/fsSSA1NYX33vuEPn36AZCfn8ef//wUu3bt5PPPP+avf32lTn2uXr2SVq1a8+WX/yEqqhMAJ06c4KmnHiMlJZl///tfPPjgI/b2NpuNl156npycbHr1iuHll/9GixYtATh06AB/+tPvOXhwf51qaGy0Z0MjZDYZuWV4+dShZVvTyD1T84kVIiIiIiJNlc1mo7jE4pBfZ8+V8fV/D13weXNXHubsuTKHPM9V9vm/6aYJTJo0BaOx/KOl0Wi0z1To3bsPV1wxsFLQABAaGsZf//oKZrOZZct+rPMzy8rK+N3vnrIHDQABAYE88cQfANi0KbZeff7lLy/agwaAVq1a8cAD06vtc8eObRw4sI9mzZrx8sv/Zw8aALp06cpzz/2VsrKyOtfRmGhmQyPVP7olUa0DSDqWz5INydw9pquzSxIRERERcRk2m43X/7WDxIy8y/bMnIJiHvnnOof01altIH++ox8Gg3P3grjYXgTFxedYvXoVu3btJDMzk3PnztqDEqPRSFpaKsXF5/Dyqv2RlH5+/owefV2V17t06YqnpydnzhSQl5dLYGBQrfvs1KkLPXv2qvJ6jx7lr2VkHK30+pYtGwEYNGgILVpU3Zj/iisG0apVOCdOHK91DY2NwoZGymAwMGVkJ974egfrdh3j2gHtaN3C19lliYiIiIi4jsazZ6PTtG/focZrR44k8fTTv+f48WMX7CM/P5+WLWsfNlxoP4agoGCysjI5e/ZsncKGmvoMCQkB4OzZokqvp6enAdCpU+ca++zUqbPCBmmcurQLom/nFuw8fIpv1yTx+OQYZ5ckIiIiIuISDAYDf76jHyWlVof0dyg9l7fn77pou99P6U2XdkGX/DxPD6PTZzUAeHt7V/u6xWLh+eef5vjxY/TvfyV33nk3nTp1xt8/wL6sYtKksWRlZdZ5uUGzZjUHExXLOeq6zKSm91HR368VFZ0FwMen5h/oXuhaU6CwoZGbPCKKXYmniU88xcG0HKIjgp1dkoiIiIiISzAYDHh5mhzSV48OIQT7e1U6heLXQvy96NEhpFEdg1mT/fv3kZqaQmhoGP/3f29VWSZhs9koKChwUnWXzsenPJwoKiqssc2FrjUF2iCykQtv7suwPuVHrsxbneQyG8mIiIiIiDQmRqOB31xT85R6gNuv6dwkggaA48czAOjWrXu1+zEcOZJUZWmCO2nXLgKApKTEGttc6FpToLChCRg/JBIvDxPJx/PZdvCks8sREREREWmU+keH8sjEngT7e1V6PcTfq1Eee3khFUsdTp8+Xe31uXO/upzlONzAgVcBsHlzLNnZVd/jtm1bL7pXRWOnZRRNQKCfF2MGRrBkQzIL1iTRt3MLzCblTCIiIiIijtY/OpS+nVtyKD2X3MJigny96NIuqMnMaKjQo0cvzGYze/YksGTJQsaPnwRAaWkps2d/xooVS/Hw8KC0tNTJldZPv34D6NatO/v37+Mvf3mal156w34qxeHDB3nttRcxm81N+vhLhQ1NxPVXtmP1zgyycs+yZmcG1wyoeQdXERERERGpP6PRQNf2TXuvtJCQ5tx++13MmfMFf//7a3zxxae0aNGSo0fTOHPmDPff/yA//vid257WYDAYeP75l3nkkQdISIhn8uRxdOwYRUlJKSkpR+jevScxMX1YuXJ5jZtMNnZN8103Qc08zUwYWn4szXexKRSda7oJm4iIiIiINLwHH3yEp576M1FRncjLy+Xo0XQ6derCyy+/wb33PuDs8i5ZRER7Pv98DmPH3kxgYCApKcmUlBRz55338O67H9lnNfj6Ns1TKQw27Rjo0iwWK9nZF9/F1Gw2EhzsS05OIWVl1R/fY7Faef6zrZzILmLs4PbcMjzK0eWKyHlqMy5F5PLSuBRxLQ05JktLSzh9+jjNm4fj4eHp0L5FauOuu6aSnHyEL76YS+fOXZxWR13HQkiILyYHLLvXzIYmxGQ0MmVEecCwIi6d7PxzTq5IRERERESk8dm7dw/JyUcICAikQ4eOzi7HKRQ2NDF9Oregc9tASsusLN6Q7OxyRERERERE3FJ6ehrz5/+HgoKCSq8nJMQzY8YzANx880TM5qa5VWLTfNdNmMFgYOrITrw6Zzuxu49z3YB2tA31c3ZZIiIiIiIibqWw8AzvvPMm7733Nu3aReDj48upUyfJysoEoFevGO699/85uUrn0cyGJiiqTSADoltis8G3a5OcXY6IiIiIiIjbad26Lb/97X106RJNXl4ehw4doLDwDD169OLxx//AO+98hJdXM2eX6TSa2dBE3TI8ip2HT5GQdJr9Kdl0iwxxdkkiIiIiIiJuIyAggGnTpjNt2nRnl+KSNLOhiQoL8WFEnzYAzFuThFWHkoiIiIiIiIiDKGxowm4aEkkzTxOpJwrYuj/T2eWIiIiIiIhII6GwoQkL8PXkhkHtAVi49gilOm9cREREREREHEBhQxN33RXtCPLz5FTeOVbvOOrsckRERERERKQRUNjQxHl5mJhwdUcAvt+YQtG5UidXJCIiIiIiIu5OYYMwpFcrWrfwpfBcGT9uTnV2OSIiIiIiIuLmFDYIJqORySOiAPhv3FFO551zckUiIiIiIiLizhQ2CAC9o5oT3S6IMouVxeuPOLscERERERERcWMKGwQAg8HAlJGdANi45wRpmQVOrkhERERERETclcIGsevYOoAru4ViA75dk+TsckRERERERMRNuV3YsHnzZh588EEGDRpETEwMY8aM4Z///CdFRUV16mfLli1ER0fX6td7771XbR+FhYW8/fbbjBkzhpiYGAYNGsSDDz7Ili1bHPFWnWLSsI6YjAb2JGezNznb2eWIiIiIiEgjNHToAIYOHVDl9UcfncbQoQPYsWNbnfrbsWMbQ4cO4NFHpzmqxIs6fvwYQ4cOYPLkmy7bM92JW4UNc+bM4Z577mHNmjV4eXkRFRVFRkYGH374IZMnTyY3N7fWffn7+9OvX78af3Xp0sXetm/fvlXuz87O5pZbbuGjjz4iIyODqKgovLy8WLNmDXfffTdff/21I97yZRca7MPIfm0AmL86EavN5uSKRERERESkIb366gsMHTqAP/zh8Vq1z84+zfDhAxk6dABxce77g9aL+fzzj/n8848pKNAS8/owO7uA2tqzZw+vvfYaAC+99BJTp07FYDCQmZnJww8/zN69e3n++eeZOXNmrfrr3r07//73v2u8/t5773Ho0CHCw8MZPHhwlevPPfccycnJ9OjRgw8//JCwsDBsNhvz5s1jxowZvPrqq/Tr149u3brV7w070U1XRRK7+zhpWWfYsjeTwT1bObskERERERFpIDfcMI6lS39g27YtnD59iubNW1yw/YoVS7FYLISGhtG//xUOrSUsrBUREe1p1qyZQ/utjy+++BSAG2+8CX9//yrXzWYzERHtadky9HKX5hbcZmbDBx98gNVqZfz48dx6660YDAYAwsLCeOuttzAajaxYsYIDBw5c8rNsNhuLFy8GYPz48RiNlb9M+/bt4+eff8ZoNPL2228TFhYGlG+yeOuttzJ+/HgsFgsffPDBJdfiDP4+ntw4qD0AC9clUVpmcXJFIiIiIiLSUPr27U94eGssFgsrViy7aPulS38EYMyYsVU+K12q559/iblzF9C9e0+H9tsQWrYMZe7cBbzzzofOLsUluUXYUFhYyPr16wGYOnVqleuRkZEMGjQIgGXLLj44LiYuLo709HQAJk2aVOX68uXLARg0aBDt27evcv3WW28FYO3atXXeS8JVXDugHcH+XpzOL2bV9gxnlyMiIiIiIg3EYDAwZsxYAJYt+/GCbQ8fPkhS0mGgfEaESE3cYhnF/v37KSkpwdPTk5iYmGrb9O/fn40bN7Jr165Lft6iRYvsfVYXJsTHxwMwYEDVDU0AYmJi8PT0pLi4mP3799O/f/9Lruly8/QwMfHqjsz6aT8/bExhaEw4ft4ezi5LRERERMTlWW1WEnOTyS/OJ8ArgE5BHTAaXPvnvGPGjGX27M9ISjrM4cMH6dw5utp2FWFEr14xtGsXwd69e1i3bjU7dsSRlZVJXl4eAQGBdO/egylTbq/zMotHH51GfPwO3n33I/r1q/x5y2q1smjRt3z33SLS09Pw8fEhJqYP9977wAX7rGuNn3/+sX0JBcCUKTdXul5R2/Hjx5gy5WZatQrn22+/r/LcwsIzfPPNXNauXU1GRjoGg4E2bdoxfPhIbr31N/j4+Fa5Z/Lkmzhx4jjvvvsRoaFhfP75x2zfHseZMwWEh7dm7Nibue22Ox0+o6QhuEXYkJycDEDr1q3x8Kj+A29ERESltvVVVFRknx0xceLEatukpKRUeuaveXh4EB4eTmpqKsnJyW4ZNgBc1bMVK+LSOHqykB83pXDrqM7OLklERERExKXFZ+1m/uHvyC3Os78W5BXIlM430ye0lxMru7A2bdoSE9OHXbt2snTpD9WGDWVlZfZlFmPGlM9qeOmlv5CRcRR//wCaN29B8+YtOXkyiw0b1hEbu57f/e4pbrnl1kuuz2az8eKLf2HVqhUAtGoVTmBgEFu2bGTz5o3ce+//q/HeutYYFtaKXr16s3t3+Q+yu3btXulzqJ+f30XrPXHiBL/73XSOHk3DaDTSoUNHAI4cSSQx8RD//e8y/vnPDwgNDav2/sOHD/LnP/+BsrIyIiM7YjabSU1N4YMP3uXEieM8+eTTF/+iOZlbhA15eeUDNTAwsMY2Fdcq2tbXsmXLKCoqwtvbmxtuuOGS68nPz7+kegDM5ounViaTsdI/HeW2azrz5r/jWbX9KNddGUHLIG+H9i/SmDXUuBSR+tO4FHEtDTkmrVaDw/u8mPis3Xy6Z06V13OL8/h0zxwe6HmXSwcON9wwjl27dvLf/y5n+vQnMJsrf1zcsmUTOTnZeHp6MXr0dQDcc8//o0ePXkREVJ4Rvn17HC+88BwzZ77NkCHDadXq0jad/+67RaxatQJPTy9efPFVrr56BABnzpzh1Vdf4PPPP67x3rrWOG7ceMaNG28/mvPll98gPLx1nep98cXnOHo0jU6duvDqq/9HmzZtAUhPT+PZZ58iOfkIL730PO+990m193/44UxuuGEcjz32JD4+PgCsWvVfXnjhWRYt+pbJk2+r8n4uxmQy1OqzpcFBQ8ctwobi4mKAGmc1AHh6elZqW18VSyiuu+66GhOrutRz7ty5S6rHaDQQHFx1ek1NAgIcGwYM6+/Df7cdZdfhU3y/MZU/3OGeszREnMnR41JELp3GpYhraYgxee6ciVOnjBf8gGWz2SixlDjkeVablfmHl1ywzfzD39GjZReHLKnwNHnaN813lGuvvY5//vPv5ORks23bZoYOHVbp+vLl5Usohg8fQVBQAAA33XRzlX4ABg4cyEMPPcLrr7/Mzz+v4Le/vafadr/+val4TyaT0X7NZrPx9ddfAnDXXXczcuQoe/ugoABeeulVJkwYS15eLgZD1d/vS63x/Fp+/Xp172PHju3s3r0Lo9HIK6+8Tvv2/5sR36FDJC+//Dp33XUb8fE72L17J337Vv2MFRHRnmeeeQ6TyWR/7frrr2flymWsX7+WrVs30rFjh2rr/TWr1YDRaCQw0OeynvLhFmGDl5cXAKWlpTW2KSkpqdS2PtLT04mLiwNqXkJR8YyzZ8/Wqp5L/c20Wm3k5198k0mTyUhAgDf5+WexWKyX9Mxfu2VYR3YdPsWaHUcZ1bc1keEBDu1fpLFqyHEpIvWjcSniWhpyTJaUFGO1WrFYbJSVVe3bZrPx1o4POJKX6tDnXkhucR6/W/28Q/rqGBjJk/0edmjg4OXlzfDho1i+/Cd+/PEHBg0aar+Wn5/Phg3rgPIlFOd/TY8dy2DlyuUcPnyIvLxc++ekwsIzABw8uL/a3wOgyus2mw0Ai8Vqv5aamsKxY+Wb1k+cOLXKPR4eXowdezNz536FzVb97/el1Hh+Lb9+vbr3ERu7AYArrxxE27btq9wbGRnFFVcMZOvWzWzcuJFevfpW6XvcuPHYbIYq93bv3pP169eSnp5eY71V67RhtVrJyyvi7NmLnzQYGOjtkD0h3CJsqM0SidosbbiYxYsXY7PZaNOmjf10i+oEBARw9uzZWtUTEHDpH8xr+00ENQ+ES9G2pR+DeoSxeW8m/155mKdu6+PwFFWkMWuIcSkil0bjUsS1NMSYtFhstWilv9P+2g03jGP58p+IjV1HQUEB/v7+APz88wpKSkpo0aIlAwZcaW8/b95cPvjgXcrKymrs81KXuqempgAQHBxCUFBQtW0q9kSozuWo8XxpaeUBVseOUTW26dixE1u3bra/t19r27b6/QGDg0MAOHv2bJ3rqil4+zVbbYZOLbhF2BAZGQnAsWPHKC0trXb5QlpaWqW2dWWz2Vi8eDEAEyZMuOCH6cjISDIzM0lNrT4FLS0t5dixY5dUj6uZdHVHth3IYn9qDnuSs+nVsbmzSxIRERERqTeDwcCT/R6mxFrzbOW6SMw9wge7Zl203fTe99EpqOYPxrXlafRokB8A9u9/BWFhrcjMPMGqVSuYMOEWAJYuLV9Ccf31N9qn9u/evYt3330Lo9HIvfc+wPDho2jdujXNmpX/ZHz79jieeOLhC37Ir42zZ8tnegcHB9fYpuJD+K9drhrPV1RUUW/Nn5lCQpr/0raw2us1zZCvmHFgc1Qi0IDcYnekbt264eHhQUlJCQkJCdW22b59OwB9+vSp1zO2bt3K0aNHMRgMF1xCcf4zKp75awkJCZSWluLl5UW3bt3qVY+raRHkzej+5ZuazF+diNXq+t/cIiIiIiIXYjAY8DJ5OuRXt5AuBHldeJZ1sFcg3UK6OOR5DTXT2GAwMGbMWOB/x1ympaWyd+9uoHzmQ4WK67feegf33/8gnTp1xsfH1/6B2FGzBby9yzdIzMnJqbFNTk52ta9frhrPV7GhY07O6RrbZGef/qVt7ffnczduETb4+fkxdGj5eqF58+ZVuZ6SksLmzZsBGDNmTL2eUbEx5IABA2jXrt0F215//fUAbNmypdrZDd988w0Aw4YNw9e38XzzjB0ciY+XmaMnC9m454SzyxERERERcRlGg5EpnavfiLDC5M43O2RzyIZWESjs2ZNAenqa/QN7t249iIz836aEx4+Xz+bu3bvqngOAPaC4VO3bRwKQm5tDbm5utW2Sk49U+/rlqvF8FadEHDmSVGObimsV760xcv3v9F9Mnz4dg8HAkiVL+Oabb+zTRrKysnjyySexWq1cc801dO3atdJ9o0aNYtSoUSxbtqzGvgsLC1m+fDkAkyZNumgtPXr0YOTIkVgsFn7/+9+TlZUFlE9l+eabb1iyZAlGo5GHH364vm/XJfl5ezDuqkgAFq0/QknpxTcXERERERFpKvqE9uKBnndVmeEQ7BXo8sdenq9t23b06tUbgKVLf2D58p+AyrMaALy8yqf6nz59qkofOTk5LF36g0PqiYhoT3h4G2w2G4sWza9y/dy5c/z003fV3nspNVYcPlDXEw8HDboKoMY9GY4cSSIubnOlto2R24QNMTExPPPMMwDMmDGDkSNHMnHiREaPHs3evXvp0KEDL7/8cpX7MjIyyMjIsK+bqc7y5cspKirCx8fHPmvhYl577TUiIyPZu3cvo0ePZuLEiYwcOZIZM2ZgMBh49tln6dGjR/3erAsb3b8NzQO8yCko5r/b0p1djoiIiIiIS+kT2ouXr/ozT/R9kHu7384TfR/kpav+7DZBQ4Ubb7wJgG+++ZrMzBN4enpyzTWVPyv16VM+W2DOnC/smyJC+ckPf/rT7zh37pxDajEYDPzmN3cB8PXXX9pPxYDy0yRefvl5zpw5U+29l1Jjmzbly8jj46tfPl+Tvn37ExPTB6vVygsvPEtGxlH7tYyMo7z44nPYbDb69OlX44yLxsAtNoiscM899xAdHc2sWbNISEjg9OnTtG7dmjFjxjBt2rR6L1moWEJx/fXX17qPkJAQFixYwKeffsqyZctITEzEx8eHYcOGcf/991/wNAt35mE2MWlYFJ/+sI+fNqcyrHdr/H08nV2WiIiIiIjLMBqMdAmu+SQCdzBq1DW8886b9g/jV111dZWT9m66aSJLliwkLS2Vu+6aSrt27TGZjCQnH8Hb25vp0x/jn/980yH1TJhwCzt2bGP16pU888yThIe3JjAwiJSUI1itNu6//0E+/vj9KvddSo3XXHM9n3zyAW+++QYLF84nIKB8xsoTT/yBzp2jL1jvjBmv8LvfPczhw4e4/fZJdOgQBdhITj6C1WqlXbsIZsyo+sPyxsStwgaAwYMHM3jw4Fq3P3jw4EXbzJkzp161+Pn58fvf/57f//739brfXQ3sEcbyrWmkZZ3h+40p/OaaLs4uSUREREREHMjX149hw0ayYsVS4H8zHc7n4+PD++9/xqeffkBs7DqOHk0jODiE6667gXvvfYDMTMft82YwGHjhhVfp3bsP33+/mPT0NM6eLeKKKwZx333TKCjIr/a+S6nxN7/5LVarlZUrl3P06FFKSsr3WSgoKLhova1ateLzz+fwn/98zdq1P5ORUT4rvEOHjowYMZpbb/1No94cEsBgc4czM5owi8VKdnb1x6Gcz2w2EhzsS05O4WU5N3xvSjb/+E88JqOBVx8YSGiwT4M/U8TdXO5xKSIXp3Ep4loackyWlpZw+vRxmjcPx8NDM3Gl6arrWAgJ8cVkuvQdF9xmzwZxLT0iQ+jZIQSL1cbCddXv/CoiIiIiIiJNk8IGqbfJI6IwAFv3Z3HkWPXTlkRERERERKTpUdgg9RYR5s9VPVsBMH91IlqRIyIiIiIiIqCwQS7RhKs7YjYZOZieS0LSaWeXIyIiIiIiIi5AYYNckuaBzbh2QPn5s/PXJGGxarMtERERERGRpk5hg1yysYPb49vMzLFThcTudtzxNiIiIiIiIuKeFDbIJfNp5sFNV0UCsHj9EYpLLM4tSERERERERJxKYYM4xMh+bWkR2IzcMyWs2Jbu7HJERERERETEiRQ2iEN4mI1MGt4RgKWbU8kvLHFyRSIiIiIiIuIsChvEYa7sFkb7Vv6cK7HwfWyKs8sREREREQF0PLs0dc4ZAwobxGGMBgNTR3YCYE18BpnZRU6uSERERESaKoOh/KOOxaLT0qRps/5yYqDBYLisz1XYIA7VrX0wMVHNsVhtLFib5OxyRERERKSJMplMGI1miovPOrsUEacqKSnGYDBiMpkv63MVNojDTR4RhcEA2w6eJCkjz9nliIiIiEgTZDAYaNbMh3PnCiktLXZ2OSJOYbVaOXeuEC+vZpd9ZsPljTakSWjb0o8hvcLZkHCceasTeeaOfpf9G1tERERExM8vkNLSYrKzs2jWzBcvL29MJiOgv5tK42az2bBYSiksLMBqteLnF3TZa1DYIA1iwtAObN2XyeGjecQfPkXfLi2dXZKIiIiINDFGo5Hg4FDOnMnj3Lkizp4tcHZJIpeVp2czAgJCMZs9LvuzFTZIgwgJaMa1V7Tjx02pzF+TREyn5piMWrUjIiIiIpeX0WgkICAYf/8gLBYLNps2jJSmwWg0YTKZnPZ8hQ3SYG4Y2J618cc4kV3E+l3HGdG3jbNLEhEREZEmymAwYDbr44/I5aIfNUuD8Wlm5uYhkQAs3pDMuZIy5xYkIiIiIiIil4XCBmlQI/q2ITTIm/zCEpZvTXd2OSIiIiIiInIZKGyQBmU2GbllRBQAy7akkXdGxw6JiIiIiIg0dgobpMENiG5Jh/AAikstLIlNcXY5IiIiIiIi0sAUNkiDMxgMTB1ZPrthXfwxjp8udHJFIiIiIiIi0pAUNshlER0RTJ9OLbDabHy7JsnZ5YiIiIiIiEgDUtggl83kEVEYDLDz8CkOpec6uxwRERERERFpIAob5LJp3cKXYb1bAzB/dSI2m83JFYmIiIiIiEhDUNggl9X4oR3w9DCSdCyf7QdPOrscERERERERaQAKG+SyCvLzYsyVEQB8uzaJMovVyRWJiIiIiIiIoylskMvu+isjCPDxICvnLGvjjzm7HBEREREREXEwhQ1y2Xl7mRk/tAMA38Umc7a4zMkViYiIiIiIiCMpbBCnuLp3a8JCfCgoKmXpljRnlyMiIiIiIiIOpLBBnMJsMjJ5eBQAK7amkVNQ7OSKRERERERExFEUNojT9OvSgk5tAikps7JkQ7KzyxEREREREREHUdggTmMwGJgysnx2w/qEY2ScKnRyRSIiIiIiIuIIChvEqTq3DaJfl5bYbLBgTZKzyxEREREREREHUNggTnfL8I4YDQbiE09xMC3H2eWIiIiIiIjIJVLYIE4X3tyX4X1aAzBvdSI2m83JFYmIiIiIiMilUNggLuHmoR3w8jCRfLyAuANZzi5HRERERERELoHCBnEJgb6e3DAwAoAFa5Mos1idXJGIiIiIiIjUl8IGcRnXXdmOQF9PTuaeY/XODGeXIyIiIiIiIvWksEFcRjNPM+Ov7gDA97EpFJ0rc3JFIiIiIiIiUh8KG8SlXB0TTnhzH86cLWXpllRnlyMiIiIiIiL1oLBBXIrJaGTyiCgAVsSlk51/zskViYiIiIiISF0pbBCX06dTC7q0DaS0zMri9cnOLkdERERERETqSGGDuByDwcCUUZ0AiN19nKNZZ5xckYiIiIiIiNSFwgZxSVGtAxnQNRQbMH9NkrPLERERERERkTpQ2CAu65bhHTEZDew+cpp9KdnOLkdERERERERqSWGDuKywYB9G9G0DwPzVSVhtNidXJCIiIiIiIrWhsEFc2k1DImnmaSI1s4Ct+zKdXY6IiIiIiIjUgsIGcWkBPp7cOKg9AAvWHqG0zOrkikRERERERORiFDaIy7v2inYE+XlyOv8cP+846uxyRERERERE5CIUNojL8/IwMfHqjgD8sDGFwnOlTq5IRERERERELkRhg7iFIb3CadPCl8JzZfy4KdXZ5YiIiIiIiMgFKGwQt2A0GpgyMgqAlduOcirvrJMrEhERERERkZoobBC30atjc7pGBFFmsbJoXbKzyxEREREREZEaKGwQt2EwGJgyshMAm/eeIPVEgZMrEhERERERkeoobBC30iE8gIHdw7AB365JdHY5IiIiIiIiUg2FDeJ2Jg3riMloYG9KDnuSTzu7HBEREREREfkVhQ3idloGeTO6f1sA5q9Owmq1ObkiEREREREROZ/CBnFL466KxNvLTHrWGTbtPeHsckREREREROQ8ChvELfl5ezB2cHsAFq0/QmmZxckViYiIiIiISAWFDeK2runflmB/L7Lzi1m5/aizyxEREREREZFfKGwQt+XpYWLSsI4A/LAxlTNnS51ckYiIiIiIiIDCBnFzg3u0om1LP84Wl/HDxhRnlyMiIiIiIiIobBA3ZzQamDoyCoCfdxzlZO5ZJ1ckIiIiIiIiChvE7fXoEEL3yGDKLDYWrTvi7HJERERERESaPIUN4vYMBgNTRnQCYPO+TFJO5Du5IhERERERkaZNYYM0Cu1b+TO4RxgA835OxGazObkiERERERGRpkthgzQaE4d1xGwycCAtl91Hsp1djoiIiIiISJOlsEEajRaB3lzTvx0A89ckYrVqdoOIiIiIiIgzKGyQRmXsVe3xbWYm42QhsXuOO7scERERERGRJsns7ALqavPmzXzxxRfs2rWLoqIiWrduzZgxY5g2bRo+Pj716tNms/Hjjz+yaNEi9u/fT35+PkFBQURFRTFs2DDuv//+KvdER0dfsM8WLVoQGxtbr3qk/nybeTB2cCTzVieyeH0yV3YLw8vD5OyyREREREREmhS3ChvmzJnDq6++is1mo1WrVoSHh5OYmMiHH37IihUrmDt3LkFBQXXqs7CwkEcffZSNGzcC0K5dO1q3bs3p06eJi4vjwIED1YYNFXr27Imnp2eV1+tahzjO6P5tWLX9KKfzz7FyWzpjB0c6uyQREREREZEmxW3Chj179vDaa68B8NJLLzF16lQMBgOZmZk8/PDD7N27l+eff56ZM2fWuk+bzcZjjz3Gxo0bufrqq5kxYwYRERH26/n5+cTFxV2wj3feeYe2bdvW701Jg/Awm5g0vCOffr+PHzelcnXv1gT4VA2EREREREREpGG4zZ4NH3zwAVarlfHjx3PrrbdiMBgACAsL46233sJoNLJixQoOHDhQ6z4XLlxIbGwsvXv35qOPPqoUNAAEBAQwevRoh74PuTwGdg8jIsyPcyUWfohNcXY5IiIiIiIiTYpbhA2FhYWsX78egKlTp1a5HhkZyaBBgwBYtmxZrfudPXs2AA8//DBms9tM8pBaMBoMTB3ZCYDVOzPIzClyckUiIiIiIiJNh1t8wt6/fz8lJSV4enoSExNTbZv+/fuzceNGdu3aVas+09LSOHToEEajkYEDB7Jr1y4WLFhAWloaPj4+9OnTh8mTJxMSEnLBfj744AOysrKwWCyEhYUxaNAgbrzxxmr3cZDLq3tkCD07hrDnSDYL1x7h4Qk9nV2SiIiIiIhIk+AWYUNycjIArVu3xsPDo9o2FUsgKtpezJ49e4DyjRy//vpr/vGPf2Cz2ezXV61axaeffsrMmTPtsyaqs2DBgkr/vWjRIt59911mzpxJjx49alWLNJwpIzqx98hW4g5kcd2xPKJaBzq7JBERERERkUbPLcKGvLw8AAIDa/6gWHGtou3FZGVlAeWbQL755puMGDGCP/7xj0RERJCcnMxrr73G5s2beeyxx/j+++9p1apVpftHjx7N+PHj6dq1K61ataKwsJBNmzbx9ttvk56ezn333cfixYsJDw+vz1uuxGy++GoXk8lY6Z9SrkPrAIbGhLM+4Tjfrkni2bv62/f7EGloGpcirkfjUsS1aEyKuB5HfVxyi7ChuLgYoMZZDYB92UJF24spKipfw19WVkZERATvvfeevf/o6Gg++ugjrr32Wk6ePMmXX37J008/Xen+Dz74oNJ/e3l5MXbsWAYPHswtt9zCsWPHeO+993j11Vdr9yZrYDQaCA72rXX7gADvS3peY3Tvzb3Ysi+Tg2m5JB4/w5U9Wl38JhEH0rgUcT0alyKuRWNSpPFxi7DBy8sLgNLS0hrblJSUVGpb2z4B7rjjjipBhre3N7fddhszZ85k/fr1VcKGmoSEhDBt2jReeOEFVq5cySuvvHJJP0m3Wm3k5198c0OTyUhAgDf5+WexWKz1fl5jZAauuzKCHzam8Pl3e+jYyheTUem5NDyNSxHXo3Ep4lo0JkVcT2CgN0YHfF5yi7ChNkskarPU4nwBAQH2f4+Kiqq2TcXrR48erVWfFfr27QtAbm4uubm5BAcH1+n+Xysrq/0fvBaLtU7tm4oxV0awZmcGx04VsmZHBsP7tHF2SdKEaFyKuB6NSxHXojEp4jrO28rwkrjFj3cjIyMBOHbsWI2zG9LS0iq1vZiOHTva/72m5RkVsx+s1rr9wXd+fxaLpU73SsPwaWbmpqsiAVi8PpniEv2+iIiIiIiINBS3CBu6deuGh4cHJSUlJCQkVNtm+/btAPTp06dWfXbv3p1mzZoBkJ6eXm2bigDj15tDXszhw4eB8rAiKCioTvdKwxnZrw0tg5qRV1jC8rg0Z5cjIiIiIiLSaLlF2ODn58fQoUMBmDdvXpXrKSkpbN68GYAxY8bUqk9vb29GjhwJwOLFi6tct9lsLFq0COCCR1/+WllZGV988YX9PrPZLVaqNAlmk5FbhpcvjVm6JY28whInVyQiIiIiItI4uUXYADB9+nQMBgNLlizhm2++wfbLQpKsrCyefPJJrFYr11xzDV27dq1036hRoxg1ahTLli2r0uejjz6K2Wxm27ZtvP/++/YlD2VlZfz973/nwIEDeHl5cc8991S6780332TRokWcOXOm0uvHjx/n8ccfJz4+HrPZzCOPPOLAr4A4woCuoXQI96e4xMJ3scnOLkdERERERKRRMthsjtr+oeHNnj2bN954A5vNRnh4OMHBwSQmJlJSUkKHDh2YO3cuISEhle6Jjo4G4PXXX2fSpElV+ly0aBHPPfccFouFkJAQ2rZtS1paGrm5uXh4ePDGG28wbty4SvdMnz6dVatWYTKZaNeuHYGBgRQUFJCcnIzNZsPLy4tXXnmFm2+++ZLfs8ViJTu78KLtzGYjwcG+5OQUanOdiziQmsP//XsnRoOBl//flYQ3r/3RoiJ1oXEp4no0LkVci8akiOsJCfHFZGoip1FUuOeee4iOjmbWrFkkJCRw+vRpWrduzZgxY5g2bRq+vnX/0Dhx4kQ6derEZ599xrZt29i/fz9BQUGMGzeOBx54oMpMCYDbb7+dFi1asGfPHrKyssjIyMDDw4POnTszePBg7rzzTiIiIhzxlqUBdG0fTO+o5uxKOs3CtUd4ZFIvZ5ckIiIiIiLSqLjVzIamSDMbGkbGyTPMmLUVmw2evas/ndrU7shUkbrQuBRxPRqXIq5FY1LE9ThqZoPb7Nkg4khtWvoxtFc4APNWJ6LMTURERERExHEUNkiTNeHqjniajSQezWPn4VPOLkdERERERKTRUNggTVawvxfXXdkOgPlrkiizaOqeiIiIiIiIIyhskCbthoHt8fP2IDO7iPUJx51djoiIiIiISKOgsEGaNG8vM+OHdgBgyfojnC0uc3JFIiIiIiIi7k9hgzR5w/u0JjTYm/yiUpZvTXN2OSIiIiIiIm5PYYM0eWaTkcnDowBYvjWd3DPFTq5IRERERETEvSlsEAH6R7ekY+sAikstfLch2dnliIiIiIiIuDWFDSKAwWBg6shOAKzbdZxjpwqdXJGIiIiIiIj7UtjQCFhtVg5mJ7IhNY6D2YlYbTrCsT66tAuib+cWWG02FqxNcnY5IiIiIiIibsvs7ALk0sRn7Wb+4e/ILc6zvxbkFciUzjfTJ7SXEytzT5NHRLEr8TQ7D5/iUHouXdoFObskERERERERt6OZDW4sPms3n+6ZUyloAMgtzuPTPXOIz9rtpMrcV3hzX4b1Dgdg3upEbDabkysSERERERFxPwob3JTVZmX+4e8u2Obbw99pSUU9jB/aAS8PE0eO5bP94ElnlyMiIiIiIuJ2FDa4qcTc5CozGn4tpziPxFydrFBXgX5eXH9lOwC+XZtEmUWBjYiIiIiISF0obHBT+cX5tWr3c/p60gsytBygjq6/MoIAX0+ycs6yNv6Ys8sRERERERFxK9og0k0FeAXUqt3uU/vYfWofLbyb0y80hr6hvWjn1waDwdDAFbo3by8z44d2YM7ygyzZkMxVPVvh7aXhIiIiIiIiUhua2eCmOgV1IMgr8IJtfM0+9G7REw+jB6fOnmZF6mr+FvcuL2z+PxYn/kRawVHNeLiAq2PCaRXiw5mzpSzdkursckRERERERNyGwaZPmy7NYrGSnV1Y7bWK0yhq8kDPu+gT2otzZcXsPX2AHVkJ7D19gFJrqb1Ni2Yh9A2NoV9oDO38NePh13YcOsl7C3fjaTby+oODCfb3cnZJ4kbMZiPBwb7k5BRSVqa9P0RcgcaliGvRmBRxPSEhvphMlz4vQWGDi7tQ2ADlgcP8w99V2iwy2CuQyZ1vpk9oryrtK4KHnVkJ7FHwcFE2m43Xv95B4tE8ro4J594buzm7JHEj+guUiOvRuBRxLRqTIq5HYUMTcbGwAcqPwUwuSKHMXIK5zJMO/pEYDRf/5ii2lJTPeMjcVWPw0De0FxH+bZt08JCYkcdrc7ZjMMCL911J25Z+zi5J3IT+AiXiejQuRVyLxqSI61HY0ETUJmyAS/+D2h48ZCWw59T+SsFD82Yh9s0lm2rw8P6i3Ww/eJKYqOb8bkpvZ5cjbkJ/gRJxPRqXIq5FY1LE9TgqbND2+gKAl8mTfr8soTg/eNh7aj+nz2Xz37Q1/DdtDc2bhdA3tBf9QmOaVPBwy/Ao4g+fIiHpNPtTc+jWPtjZJYmIiIiIiLgszWxwcZdrZkNNKoKHnb/MeCipNOMh2L7HQ1MIHv614iA/78ggspU/f7l7AMZG/n7l0umnNSKuR+NSxLVoTIq4Hs1skMvi/BkPJZYS9pwXPJw+l8PKtLWsTFtL82bB9PllxkN7/3aNMni4eUgHYvecIOVEAXH7sxjYPczZJYmIiIiIiLgkzWxwcc6e2VCTEksJe08fZGdWArtP7as04yGkWbB9qUVjCx6+j01m0fpkWgQ249UHBuFhvvTETxov/bRGxPVoXIq4Fo1JEdejmQ3iVJ4mT/qG9qJvaK/KwcPp/WSfy2FV2jpWpa0rDx5a9qJvaAyRAe4fPFx3RQQ/78zgVN45Vu/M4Lor2jm7JBEREREREZejmQ0uzlVnNtSkxFLCvtMH2fFL8FBiKbFfC/YK+uVUC/cOHtbtOsbspQfwbWbmbw8NxqeZh7NLEhflKuNSRP5H41LEtWhMirgeHX3ZRLhb2HC+Eksp+3451aK64KFiqUVkQIRbBQ8Wq5UZn2/l+OkibhzUnskjopxdkrgoVxyXIk2dxqWIa9GYFHE9ChuaCHcOG85XYillX3b5UouEU/vcPniIP3yKdxck4GE28vq0QYQENHN2SeKCXH1cijRFGpcirkVjUsT1aM8GcSueJg/6tOxJn5Y9KwUPu0/tI6c4l5/T1/Nz+np78FCx1MJocM0NGHt3ak6XdkEcSs9l0foj3D+2u7NLEhERERERcRma2eDiGsvMhpqUWErZn/3LHg+n9lF83oyHIK/ASns8uFrwcORYPq98tQ0D8MJ9V9Iu1M/ZJYmLcddxKdKYaVyKuBaNSRHXo2UUTURjDxvOd7Hg4fylFq4SPHy4eA9xB7Lo2TGEJ6f2cXY54mIaw7gUaWw0LkVci8akiOvRMgppdDxNHvRu2ZPeLXtSaillX/YhdmTtYs+p/eQW57E6fQOr0ze4VPBwy/CO7Dh0kj1Hstmbkk2PyBCn1SIiIiIiIuIqNLPBxTWlmQ01qQgeKvZ4OGcptl8L8gqkb8te9AtzXvAw97+HWLn9KBFhfsy45wqMbrDBpVwejXlcirgrjUsR16IxKeJ6NLNBmgwPkwe9W/agd8selFpK2Z99yL7UIrc4j9VHN7D66AZ78NA3NIYOgZcveBg3JJLYPcdJyzzDln2ZDO7R6rI8V0RERERExFVpZoOL08yGmv0veNj9y4yHc/ZrQV6B9GnZk36hvS9L8PDjphQWrD1C84BmvDZtIB5mU4M+T9xDUxyXIq5O41LEtWhMirgezWyQJs/D5EFMyx7EtOxBqbWMA9mH2J75vxkPa47GsuZorD146BsaQ8fA9g0SPFwzoB0/78jgdP45Vm3PYMzACIc/Q0RERERExF1oZoOL08yGuqsIHnZkJZBwsvKMh0DPAPr8srmko4OH9QnH+OKnA/h4mXnjocH4eXs4rG9xTxqXIq5H41LEtWhMirgezWwQqYGH0UyvFt3p1aJ7leAhrySftUdjWXs01uHBw5Ce4ayISyfjZCE/bUpl6qhODnpHIiIiIiIi7kUzG1ycZjY4TkXwsDNrNwmn9nK27PwZD/6/BA+9Lyl4SEg6zT/n78JsMvDaA4NoEeTtqPLFDWlcirgejUsR16IxKeJ6NLNBpI5+PePhYPbh8hkPp/aSV1LA2qMbWXt0oz146NsyhqigyDoFD706htCtfTD7U3NYtP4ID9zUowHfkYiIiIiIiGvSzAYXp5kNDe/XwcP5Mx4CPP3p07J8qUVtg4eUE/m8NHsbAH+95wrat/JvsNrFtWlcirgejUsR16IxKeJ6HDWzQWGDi1PYcHmVWcs4cNHgoRdRQR0uGDx88t1eNu/LpFv7YJ66rQ8Gg+FylC8uRuNSxPVoXIq4Fo1JEdejZRQiDcBsNNOzRTd6tuhmDx52Zu1m16m95JcUsC5jI+syNv4SPPT8ZcZD1eBh4rCObDuYxf7UHPYmZ9OzY3MnvSMREREREZHLT2GDSA3ODx5ut5ZxMCeRHZkJ5wUPm1iXsQl/Tz/6tuxF39AYOv0SPLQM8mZUv7asiEtn3uokukeGYDRqdoOIiIiIiDQNChtEasFsNNOjeVd6NO/6v+AhK4GEk3spKDlTKXio2OPhxsERbEg4ztGTZ9i09wRDeoU7+22IiIiIiIhcFtqzwcVpzwbXVmYt42BOEjuydpFwci9FZWft1/w9/QixRnJotzcBhPPGtMF4epicWK1cbhqXIq5H41LEtWhMirget9mzwWKx8O9//5vY2FiMRiMjRoxgypQpDf1YkcuifMZDND2aR1MWXR487MxKYNfJPRSUnKGAPXh1g3Olnry9KYmJvYbQKahjnY7TFBERERERcTcOmdnw7bff8vzzz3P99dfzz3/+s9K1J554ghUrVgBgs9kwGAyMGTOGt99++1If2yRoZoN7slgt9qUW208kUGIrtl/z9/Cjd2hP+rUs3+PBZNRsh8ZK41LE9WhcirgWjUkR1+NSMxtiY2MBGDduXKXXt2zZwvLlywHo168fzZo1Y9OmTSxbtoyxY8dyzTXXOOLxIi7HZDTRvXk03ZtHc2uXicz45ieyjSk0a3mSgtIzbMjYzIaMzfh5+P5yqkVvBQ8iIiIiItJoOGQu9/79+4HyQOF8ixcvBmDq1KnMnTuXWbNm8dhjj2Gz2Vi0aJEjHi3i8jxMZu4cPJTSlJ4U7RjBnZ3u4qrwK/A1+3CmtJANx7bwbvwnPBv7Cv8+sIAD2YexWC3OLltERERERKTeHDKzIScnB09PT0JCQiq9vmnTJgwGA3fddZf9tTvuuIN3332XPXv2OOLRIm6hR4cQenQIYW9yNrt2Gnho/BRui57EoZwkdvyyx0NF8LDh2Bb8PHzp3bIn/UJj6BzUUTMeRERERETErTgkbCgsLMTHx6fSa1lZWZw4cYIWLVrQuXNn++uBgYH4+fmRnZ3tiEeLuI0pI6LYl5zN1v1ZXH9lPh3CA+jWvAvdmnfhtuiJ/wseTpUHD7HHthCr4EFERERERNyQQ8IGPz8/8vLyOHv2LN7e3gDExcUB0Ldv32rv8fLycsSjRdxGRJg/g3u2YuOeE8z7OZE//aYvBoMBKN/jwR48WCdyKLf8VIv4k9UFDz3oF9pbwYOIiIiIiLgsh4QNnTt3Ztu2bSxdupRJkyYB5fs1GAwGrrjiikptCwoKOHPmDJGRkY54tIhbmXh1R7buz+Jgei4JSafp3alFlTYmo4luIV3oFtKFW7tUFzxsJfbYVnvw0Dc0hi5BUQoeRERERETEZTgkbBg3bhxxcXG89NJL7Nq1i1OnTrF+/Xo8PT254YYbKrXduXMngMIGaZKaBzbjmgFtWbYljW/XJNGrY3OMRkON7X8dPBzOPVJpj4eK4MHXw4feLXrSL0zBg4iIiIiIOJ9DwobJkyezfPlyNm7cyLx587DZbBgMBn73u9/RsmXLSm2XLVtW7YwHkaZi7OD2rN91jIxThcTuPs7VvVvX6j6T0UTXkM50DenMrV0mVAkeNh7fysbj5wUPoTF0CVbwICIiIiIil5/BZrPZHNGR1Wrlhx9+YOfOnQQEBDBs2DD69+9fqU1JSQmPPPII586dY8aMGZU2jpTqWSxWsrMLL9rObDYSHOxLTk4hZWXWy1CZXIrlW9P45udEgvw8ef3BwXh51D8QsFgtHM49UmmpRYXy4KF8jwcFD5efxqWI69G4FHEtGpMirickxBeTyXjJ/TgsbJCGobChcSots/Lcp5s5lXeOScM6Mu6qSIf0e8Hgwexj3+MhOriTgofLQONSxPVoXIq4Fo1JEdejsKGJUNjQeG3ee4JPvt9HM08Tbzw0mAAfT4f2b7FaSMxNZsfJBOKzdlcJHmJa9qCfgocGpXEp4no0LkVci8akiOtxq7Bh9erVxMbGYjQaGT58OEOGDGnoRzYaChsaL6vNxsuzt5GaWcDo/m2549ouDfas84OHXVl7KCg9Y79WETz0DY2hq4IHh9K4FHE9GpcirkVjUsT1uFTYsGLFCv72t78xZMgQXnrppUrXXn/9db766qtKr91zzz08/fTTl/rYJkFhQ+O2PyWbv/8nHpPRwCsPDCQs2KfBn2m1WUnMPcKOrN3EZ+2uFDz4mL1/mfHQW8GDA2hcirgejUsR16IxKeJ6HBU2OOQ0ip9//pljx44xYMCASq/v3buXL7/8EoDWrVvj4eFBamoqs2fPZsSIEQwcONARjxdxW90iQ+jVsTm7j5xmwdojTJ/Qs8GfaTQY6RLciS7BnZjaZXyV4GHz8W1sPr7tvOChfKmF2eiQPy5ERERERKQJcMinh927dwMwePDgSq8vWLAAgGuvvZZ33nkHo9HIyy+/zNdff828efMUNogAU0ZEsefIabYdyCLpWB5RrQMv27OrBg/J7MhKIP7kbgpKFDyIiIiIiEj9OOQTQ3Z2NiaTiZYtW1Z6PTY2FoPBwAMPPIDRWD4N48EHH+Trr78mPj7eEY8WcXttQ/0Y0iucDbuPM//nRJ6+ox8Gg+Gy11EePETRJTjKHjzszEpg56+CB2+zN71b9KBvaC+6hnRW8CAiIiIiIlU45FNCQUEBvr6+lV7LyckhNTWVwMBAYmJi7K+Hhobi7e3NyZMnHfFokUZhwtUd2LI/k0NH84hPPEXfzi0vflMDOj94mNJlPEm/zHiwBw8ntrH5hIIHERERERGpnkM+Gfj4+FBQUEBpaSkeHh4AbN++HYA+ffpUaV/RRkTKhQQ047or2vHjplS+XZNETFRzTMZL35TFEYwGI52Do+hcKXjYTfzJ3eSXFFQKHmJadKdfaIyCBxERERGRJs4hnwY6duzIrl27WLt2Lddccw0AS5cuxWAw0L9//0ptz549S0FBAe3atXPEo0UajRsGtmdt/DGOny5ifcJxRvRp4+ySqqgcPNxMUm6KfY+H/JICtpzYzpYT2/E2NyOmRQ8FDyIiIiIiTZRDPgFce+21xMfH85e//IUjR45w8uRJfvrpJ4xGIzfccEOltrt378Zms9G2bVtHPFqk0fBpZuamIZH8e+VhlqxPZlD3MJp5uu6H9PLgoSOdgzvag4edJxOIz9pNXjXBQ/lSiy54KHgQEREREWn0HPK3/jvvvJPvvvuOgwcP8vbbb2Oz2eyv/3oGw4oVKzAYDFWOyRQRGNm3DSu3pXMy9xwrtqZz89AOzi6pVs4PHiZ3vpkjeansyNql4EFEREREpIky2CqSgUtUWFjIl19+SXx8PP7+/owcOZJx48ZValNSUsKUKVMoKCjgn//8Z6WNI6V6FouV7OzCi7Yzm40EB/uSk1NIWZn1MlQmDWXr/kw+WrIXLw8Tbzw0mEBfT2eXVG9Wm/WX4CGB+KwE8koK7Ne8zc3oZd/joXEGDxqXIq5H41LEtWhMiriekBBfTKZL3z/OYWGDNAyFDU2PzWbjla+2kXy8gJF923DX9dHOLskhKgcPu8krybdfa2ZqRkzLxhc8aFyKuB6NSxHXojEp4nqabNiwefNmvvjiC3bt2kVRURGtW7dmzJgxTJs2DR8fn3r1abPZ+PHHH1m0aBH79+8nPz+foKAgoqKiGDZsGPfff3+19xUWFvLJJ5+wfPlyjh07ho+PD7179+a+++5j4MCBl/I27RQ2NE0H03L429ydGA0GXv5/VxLe3PfiN7mRiuBhZ1YCO6sJHspnPPSiW0gXPEzue3qNxqWI69G4FHEtGpMirselw4YzZ86wb98+Tp8+DUDz5s3p3r07fn5+l9TvnDlzePXVV7HZbLRq1YqQkBASExMpKSkhKiqKuXPnEhQUVKc+CwsLefTRR9m4cSMA7dq1IygoiNOnT5OZmYm/vz9btmypcl92dja/+c1vSE5OxtPTk06dOpGdnc2JEycwGAw8//zz3HHHHZf0fkFhQ1P27rcJxCeeol+Xljw6qZezy2kwVpuV5Lw0dmTtqiZ48KJXix5uGzxoXIq4Ho1LEdeiMSnielwybKjYIHL9+vVYrZX/sDAajQwfPpwnnniC6Oi6Twvfs2cPU6ZMwWaz8eKLLzJ16lQMBgOZmZk8/PDD7N27l+uuu46ZM2fWuk+bzcb9999PbGwsV199NTNmzCAiIsJ+PT8/n7i4OEaPHl3l3ocffpiff/6ZHj168OGHHxIWFobNZmPevHnMmDEDk8nEggUL6NatW53f6/kUNjRdGacKmfH5Fmw2+POd/ejcNsjZJTW4iuBhZ1YCO0/uJrc4z36tPHjoTt/QGLq7SfCgcSniejQuRVyLxqSI63G5sGHFihX88Y9/pKSkhJq6NBgMeHp68uabb3LttdfWqf/p06ezatUqJkyYwN/+9rdK11JSUrjhhhuwWq0sWbKErl271qrPBQsW8Oyzz9K7d2/mzp2L2Vy7deL79u1j4sSJGI1Gli1bRvv27Std/9Of/sSSJUvqHH5UR2FD0zZ76QHW7TpGVJsAnr2zPwaDwdklXTZWm5WU/DR2ZLpv8KBxKeJ6NC5FXIvGpIjrcVTY4JBd2NLT03nqqacoKSmhTZs2/L//9/8YMmQIrVq1AuDEiRPExsby+eefc/ToUZ566il++OGHKsdi1qSwsJD169cDMHXq1CrXIyMjGTRoEBs3bmTZsmW1Dhtmz54NlM9SqG3QALB8+XIABg0aVCVoALj11ltZsmQJa9eupaioqN57SYiMH9qBzftOkJSRz45DJ+kfHerski4bo8FIx8BIOgZGMqnzuPLg4Zc9HnKL84jL3Elc5k6ambzo2aIb/UJj6B4S7bLBg4iIiIhIU+KQsOHzzz+npKSEPn368Pnnn+PrW3kzu4iICCIiIhg/fjz33Xcfu3bt4osvvmDGjBm16n///v2UlJTg6elZ43GZ/fv3Z+PGjezatatWfaalpXHo0CGMRiMDBw5k165dLFiwgLS0NHx8fOjTpw+TJ08mJCSkyr3x8fEADBgwoNq+Y2Ji8PT0pLi4mP3799O/f/9a1STya8H+Xlx/RQTfb0zh2zVJ9O7UArMDUkZ3Uyl46DSOlPx0+x4PucV5bMuMZ1tmPF4mT/txmgoeREREREScxyFhw6ZNmzAYDLz44otVgobz+fj48OKLLzJ+/HhiY2Nr3X9ycjIArVu3xsOj+g8PFXstVLS9mD179gAQFBTE119/zT/+8Y9Kyz9WrVrFp59+ysyZMxk0aFCle1NSUio989c8PDwIDw8nNTWV5ORkhQ1yScYMjGBNfAaZOWdZt+sYo/q1dXZJTlUePLSnY2B7e/BQcapFTnFutcFDt5BoPBU8iIiIiIhcNg4JG06cOIGvr2+tNn6Mjo7Gz8+PEydO1Lr/vLzytdqBgYE1tqm4VtH2YrKysoDyTSDffPNNRowYwR//+EciIiJITk7mtddeY/PmzTz22GN8//339iUhda0nPz+/xja1ZTZf/CfZFWtqHLG2RlyLv9mTicM68tWygyzZkMzVvVvj7eWQodsIGOnSvANdmndgStebSMlLZ1vmLnZkJpBz7vzgwYuYlt3pHxZDjxZdL1vwoHEp4no0LkVci8akiOtx1DZxDvnEYjabKSsrq1Vbm81GaWlpnfZIKC4uBqhxVgOAp6dnpbYXU1RUBEBZWRkRERG899579v6jo6P56KOPuPbaazl58iRffvklTz/9dL3qOXfuXK3qqYnRaCA4uObZIr8WEOB9Sc8T1zRxVBdWbjvKsVOF/Bx/jDvHXNopJ41V85Du9O/QHavtVhJPp7A5fQebju7gdFEOcSd2EndiJ83MXvRr3Yur2vWnT6vueJo9G7wujUsR16NxKeJaNCZFGh+HhA3t27dn//79rF+/nquvvvqCbdevX09xcTFRUVG17t/LywuA0tLSGtuUlJRUalvbPgHuuOOOKsGBt7c3t912GzNnzmT9+vWVwgYvLy/Onj1bq3qaNWtWq3pqYrXayM8vumg7k8lIQIA3+flnsVi0k29jdMuIKGZ+m8CiNYlc1T2MYP/afa83VS1NYdwUeQPj2o8hJS/NPuMh+1wuG9O2sTFt2y8zHrrRL6w3PRtgxoPGpYjr0bgUcS0akyKuJzDQG6PRRU6jGDVqFPv27eP555/n888/rzFISExMZMaMGRgMBkaPHl3r/muzRKI2SxvOFxAQYP/3muqteP3o0aNV7j179myt6jn/OfVVl2OALBarjg1qpPpENSeqTQBJGfksWJPEPTfU7tQVgXZ+7Wjn144JHcfa93jYkZVATnEucSfiiTsRj6fJk17NfznVorljgweNSxHXo3Ep4lo0JkVcx3lbGV4Sh4QN99xzD/Pnz+fEiRNMmDCBMWPGMHjwYMLCwoDyPR02bdrE8uXLKS0tpVWrVtx999217j8yMhKAY8eOUVpaWu3yhbS0tEptL6Zjx472f69pOUTF7AertfIffJGRkWRmZpKamlrtfaWlpRw7dqxO9YhcjMFgYOrITrz+rx2sTzjGdVe0o3WL2i+xkfKvYYfACDoERjCx01hSC9LZkfm/4GF71i62Z+2yBw99Q2Po0TwaT1PDL7UQEREREWlMHBI2+Pn58dlnn/HQQw+RkZHBDz/8wA8//FClnc1mo23btnz44Yf4+fnVuv9u3brh4eFBSUkJCQkJ1Z7usH37dgD69OlTqz67d+9Os2bNOHfuHOnp6VVOnID/BRjnbw5Z8YwtW7bYn/lrCQkJlJaW4uXlRbduWlsvjtO5bRB9O7dg5+FTfLsmiccnV38UrFycwWAgMiCCyIDzgodfTrXIPpdTKXjo2bwr/UJ7K3gQEREREaklh2372rlzZ7777juefPJJunXrhtFoxGazYbPZMBqNdOvWjaeeeoolS5bQuXPnOvXt5+fH0KFDAZg3b16V6ykpKWzevBmAMWPG1KpPb29vRo4cCcDixYurXLfZbCxatAigShBx/fXXA7Bly5ZqZzd88803AAwbNuyCR4GK1MfkEVEYDQbiE09xMC3H2eU0ChXBw6RO43hp8DP8ccCjjI4YRkizYEosJezISuCzPXN4ev2LfL7nX+zISqDEUnLBPq02KwezE9mQGsfB7ESsNk0NFREREZGmw2CzOWpFRmWlpaWV9lGoWKpQUFDAb3/7WwwGAwsXLqx1fwkJCUydOhWAF198kalTp2IwGMjKyuKhhx5i7969XHPNNbz//vuV7hs1ahQAf/rTn6oEEYmJiYwfP56ysjIef/xxHnroIUwmE2VlZbz11lt8/vnneHl5sXjx4krLLgAeeughVq9eTY8ePfjoo48IDQ3FZrMxb948ZsyYgdFo5Ntvv6VHjx51+8L9isViJTu78KLtzGYjwcG+5OQUar1bE/DV8oOs2ZlBh/AA/vLb/hgcdT6NVGKz2UgrOMqOX/Z4yD73v3DH0+hBjxblezz0bN610oyH+KzdzD/8HbnF/9vXJcgrkCmdb6ZPaK/L+h5EpDL9/1LEtWhMiriekBBfhxxH22BhQ01ycnIYPHgwBoOB/fv31+ne2bNn88Ybb2Cz2QgPDyc4OJjExERKSkro0KEDc+fOJSQkpNI90dHRALz++utMmjSpSp+LFi3iueeew2KxEBISQtu2bUlLSyM3NxcPDw/eeOMNxo0bV+W+7Oxsbr/9dlJSUvD09KRTp07k5ORw/PhxDAYDzz33HHfddVed3l91FDZIdfLOFPPMx5spLrXw8ISeXNE11NklNXrnBw87sxI4XUPwYLFamL3v3zX280DPuxQ4iDiR/n8p4lo0JkVcj6PCBofs2XC53HPPPURHRzNr1iwSEhI4ffo0rVu3ZsyYMUybNq1eSxYmTpxIp06d+Oyzz9i2bRv79+8nKCiIcePG8cADD9C1a/U7/oeEhLBgwQI+/fRTli1bRmJiIj4+PgwbNoz777+/2j0gRBwl0M+LMQMjWLIhmQVrkujbuQVmB/yBIDUzGAy0D2hH+4B2TIi6kbSCo+zM2s2OrF2cPpfDzl9CiIv59vB3xLTsgdGg3y8RERERabzcamZDU6SZDVKTcyVlPPPxZvILS/jNNZ25ZkA7Z5fUJFXMeNiZtZstx7eTX1pw0Xue6PsgXYKrP3JXRBqW/n8p4lo0JkVcj6NmNuhHayJuqpmnmQlDOwDwXWwKRefKnFxR01Qx42FCpxuZ1Lnqkqvq5BfnN3BVIiIiIiLOpbBBxI1d3TucViE+nDlbytItVU9Gkcsr0CugVu0CatlORERERMRdKWwQcWMmo5EpI8qn46+ISyc7/5yTK2raOgV1IMgr8IJtgr0C6RTU4TJVJCIiIiLiHAobRNxcn84t6Nw2kNIyK4s3JDu7nCbNaDAypfPNF2wzufPN2hxSRERERBo9/Y1XxM0ZDAamjuwEQOzu4xzNOuPkipq2PqG9eKDnXVVmOAR7BerYSxERERFpMup19GW3bt0cXYeIXIKoNoEMiG7JtoMn+XZtEr+b0tvZJTVpfUJ7EdOyB8kFKZSZSzCXedLBP1IzGkRERESkyajX33xtNtsl/RIRx7tleBQmo4GEpNPsT8l2djlNntFgJDqkE0PbX0F0SCcFDSIiIiLSpNRrZsOjjz7q6DpE5BKFhfgwok8bVu04yrw1STx/dzBGg8HZZYmIiIiISBOksEGkEblpSCSxe46TeqKArfszGdS9lbNLEhERERGRJkjzekUakQBfT24Y1B6AhWuPUFpmdXJFIiIiIiLSFClsEGlkrruiHUF+npzKO8fqHUedXY6IiIiIiDRBChtEGhkvDxMTru4IwPcbUyg8V+rkikREREREpKlR2CDSCA3p1Yo2LXwpPFfGT5tSnV2OiIiIiIg0MQobRBohk9HI5BFRAPx321FO551zckUiIiIiItKUKGwQaaRioprTNSKIMouVReuPOLscERERERFpQhQ2iDRSBoOBKSM7AbBpzwnSMgucXJGIiIiIiDQVChtEGrEO4QFc2S0UGzB/TZKzyxERERERkSZCYYNIIzdpeBQmo4G9ydnsST7t7HJERERERKQJUNgg0siFBnkzql9bAOavTsJqszm5IhERERERaewUNog0ATcNicTby0x61hk27z3h7HJERERERKSRU9gg0gT4eXswdnB7ABatO0JpmcXJFYmIiIiISGOmsEGkibimf1uC/b04nV/Mqu0Zzi5HREREREQaMYUNIk2Ep4eJiVd3BOCHjSmcOVvq5IpERERERKSxUtgg0oRc1bMVbVv6UlRcxo+bUpxdjoiIiIiINFIKG0SaEKPRwJSRnQBYtf0op3LPOrkiERERERFpjBQ2iDQxPTuE0K19MGUWGwvXH3F2OSIiIiIi0ggpbBBpYgwGA1N/md2weW8mqScKnFyRiIiIiIg0NgobRJqg9q38GdQjDIB5qxOx2WxOrkhERERERBoThQ0iTdSkqztiNhnYn5rDnuRsZ5cjIiIiIiKNiMIGkSaqRZA3o/u3BWD+6kSsVs1uEBERERERx1DYINKEjR0ciY+XmaMnC9m454SzyxERERERkUZCYYNIE+bn7cG4qyIBWLT+CCWlFucWJCIiIiIijYLCBpEmbnT/NjQP8CKnoJj/bkt3djkiIiIiItIIKGwQaeI8zCYmDYsC4KfNqRQUlTi5IhERERERcXcKG0SEgT3CiAj142yxhe83pji7HBERERERcXMKG0QEo8HAlFGdAFi9I4OsnCInVyQiIiIiIu5MYYOIANAjMoSeHUKwWG0sXHfE2eWIiIiIiIgbU9ggInaTR0RhALbuz+LIsXxnlyMiIiIiIm5KYYOI2EWE+XNVz1YAzF+diM1mc3JFIiIiIiLijhQ2iEglE4d1xGwycjA9l11Jp51djoiIiIiIuCGFDSJSSUhAM669oi0A365JwmK1OrkiERERERFxNwobRKSKsYPa49vMzLFThcTuPuHsckRERERExM0obBCRKnyaeXDTkA4ALFp/hOISi5MrEhERERERd6KwQUSqNbJvG1oENiPvTAkr4tKcXY6IiIiIiLgRhQ0iUi0Ps5FbhkcB8NOWNPILS5xckYiIiIiIuAuFDSJSoyu6hRLZyp/iEgvfxSY7uxwREREREXETChtEpEZGg4EpIzsBsDb+GCeyi5xckYiIiIiIuAOFDSJyQd3aBxMT1RyL1caCtUnOLkdERERERNyAwgYRuajJI6IwGGD7wZMkZuQ5uxwREREREXFxChtE5KLatvRjSK9wAOavTsRmszm5IhERERERcWUKG0SkViYM7YCn2cjho3nEHz7l7HJERERERMSFKWwQkVoJCWjGtVe0A2D+miQsVquTKxIREREREVelsEFEau2Gge3x8/bgRHYR63cdd3Y5IiIiIiLiohQ2iEit+TQzc/OQSAAWb0jmXEmZcwsSERERERGXpLBBROpkRN82hAZ5k19YwvKt6c4uR0REREREXJDCBhGpE7PJyC0jogBYtiWNvDPFTq5IRERERERcjcIGEamzAdEt6RAeQHGphSWxKc4uR0REREREXIzCBhGpM4PBwNSR5bMb1sUf4/jpQidXJCIiIiIirkRhg4jUS3REMH06tcBqs/HtmiRnlyMiIiIiIi5EYYOI1NvkEVEYDLDz8CkOpec6uxwREREREXERChtEpN5at/BlWO/WAMxfnYjNZnNyRSIiIiIi4goUNojIJRk/tAOeHkaSjuWz/eBJZ5cjIiIiIiIuQGGDiFySID8vxlwZAcC3a5Mos1idXJGIiIiIiDibwgYRuWTXXxlBgI8HWTlnWRt/zNnliIiIiIiIkylsEJFL5u1lZvzQDgB8F5vM2eIyJ1ckIiIiIiLOpLBBRBzi6t6tCQvxoaColKVb0pxdjoiIiIiIOJHCBhFxCLPJyOThUQCs2JpGTkGxkysSERERERFnUdggIg7Tr0sLOrUJpKTMypINR5xdjoiIiIiIOInCBhFxGIPBwNSRnQBYn3CcjJNnnFyRiIiIiIg4g8IGEXGoTm0D6d+lJTYbfLsmydnliIiIiIiIEyhsEBGHu2VEFEaDgV1JpzmQmuPsckRERERE5DIzO7uAutq8eTNffPEFu3btoqioiNatWzNmzBimTZuGj49Pnfp65plnWLRo0QXbfPrppwwbNqzK69HR0Re8r0WLFsTGxtapHpHGolWID8P7tmb1jgzmrU7kL3cPwGgwOLssERERERG5TNwqbJgzZw6vvvoqNpuNVq1aER4eTmJiIh9++CErVqxg7ty5BAUF1bnf8PBwwsPDq70WGBh4wXt79uyJp6dnldfrU4dIY3LzkA5s3HOClBMFbDuQxZXdwpxdkoiIiIiIXCZuEzbs2bOH1157DYCXXnqJqVOnYjAYyMzM5OGHH2bv3r08//zzzJw5s85933LLLTz22GP1quudd96hbdu29bpXpDEL9PXkhoERLF6fzLdrkujbuSUeZq3cEhERERFpCtzmb/4ffPABVquV8ePHc+utt2L4ZUp2WFgYb731FkajkRUrVnDgwAEnVyoiFa6/IoJAX09O5Z1jzc4MZ5cjIiIiIiKXiVuEDYWFhaxfvx6AqVOnVrkeGRnJoEGDAFi2bNllrU1EaublaWLC1R0A+H5jCkXnSp1ckYiIiIiIXA5usYxi//79lJSU4OnpSUxMTLVt+vfvz8aNG9m1a1ed+9+yZQuHDx8mNzeXgIAAevTowc0330ybNm0ueu8HH3xAVlYWFouFsLAwBg0axI033ljtPg4iTdHQmHBWxKVz/HQRP21OY/KIKGeXJCIiIiIiDcwtwobk5GQAWrdujYeHR7VtIiIiKrWti7i4uEr//d///pf333+fJ554ggceeOCC9y5YsKDSfy9atIh3332XmTNn0qNHjzrXItLYmIxGJo+IYuaC3fx3Wzqj+rUhJKCZs8sSEREREZEG5BZhQ15eHnDhkyEqrlW0rY327dvzzDPPMGjQINq0aYOnpycHDx5k1qxZLFu2jDfffBMfHx/uuOOOKveOHj2a8ePH07VrV1q1akVhYSGbNm3i7bffJj09nfvuu4/FixfXeMpFXZhrsameyWSs9E8RVzKgayjR7YI4mJ7LkthkHripaQRxGpcirkfjUsS1aEyKuB5HnVhvsNlsNsd01XDef/993n33XQYMGMDXX39dbZtNmzZxzz33YDKZ2Ldv3yU/88UXX2Tu3LkEBASwZs0afH19a3VfdnY2t9xyC8eOHWPy5Mm8+uqrl1SHzWazb4Yp4s4Opmbz1LvrMRjg3T+MJDI8wNkliYiIiIhIA3GLmQ1eXl4AlJbWvLlcSUlJpbaX6sknn2T+/Pnk5+ezefNmRo8eXav7QkJCmDZtGi+88AIrV67klVdeuaSwwGq1kZ9fdNF2JpORgABv8vPPYrFY6/08kYYSGuDFld1C2bo/i08XJfDU7X2dXVKD07gUcT0alyKuRWNSxPUEBnpjNF76bCO3CBtqs0SiNkst6sLf35/OnTuzb98+UlNT63Rv377lH6Jyc3PJzc0lODj4kmopK6v9H7wWi7VO7UUup4nDOrL94EkSkk6TkHiK7pEhzi7pstC4FHE9GpcirkVjUsR1OGrtg1ssjoqMjATg2LFjNc5uSEtLq9TWESo2oywrK6vXfQAWi8Vh9Yi4u7BgH0b0LT/lZf7qJKyuv4pLRERERETqwS3Chm7duuHh4UFJSQkJCQnVttm+fTsAffr0ccgzy8rKOHLkCACtWrWq072HDx8Gypd0BAUFOaQekcbipiGRNPM0kZpZwNZ9mc4uR0REREREGoBbhA1+fn4MHToUgHnz5lW5npKSwubNmwEYM2aMQ575zTffUFBQgNlsZtCgQbW+r6ysjC+++AKAQYMGYTa7xUoVkcsmwMeTGwe1B2DB2iOUasqkiIiIiEij4xZhA8D06dMxGAwsWbKEb775hopDNLKysnjyySexWq1cc801dO3atdJ9o0aNYtSoUSxbtqzS67Gxsfz9738nJSWl0uslJSXMmTOH119/HYDbbruN0NDQSm3efPNNFi1axJkzZyq9fvz4cR5//HHi4+Mxm8088sgjjnjrIo3OtVe0I8jPk9P55/h5x1FnlyMiIiIiIg7mFkdfVpg9ezZvvPEGNpuN8PBwgoODSUxMpKSkhA4dOjB37lxCQipvOBcdHQ3A66+/zqRJk+yvr1y50h4GtGjRgrCwMACSk5MpKio//eH666/nzTffxNPTs1Kf06dPZ9WqVZhMJtq1a0dgYCAFBQUkJydjs9nw8vLilVde4eabb77k92yxWMnOLrxoO7PZSHCwLzk5hdpcR9zC+l3H+GLpAXybmXnjocH4NvO4+E1uRuNSxPVoXIq4Fo1JEdcTEuKLydRETqOocM899xAdHc2sWbNISEjg9OnTtG7dmjFjxjBt2jR8fX1r3VePHj2YPn068fHxpKamkpycTGlpKSEhIQwdOpSJEycyatSoau+9/fbbadGiBXv27CErK4uMjAw8PDzo3LkzgwcP5s477yQiIsJRb1ukURrSK5wVcelknCrkx02pTB3ZydkliYiIiIiIg7jVzIamSDMbpDFLSDrFP+cnYDYZeW3aQFoEeju7JIfSuBRxPRqXIq5FY1LE9ThqZoPb7NkgIo1Pr47N6RoRRJnFyqJ1yc4uR0REREREHERhg4g4jcFgYMovyyc27z1B6okCJ1ckIiIiIiKOoLBBRJyqQ3gAA7uHYQO+XZPo7HJERERERMQBFDaIiNNNGtYRk9HA3pQc9iSfdnY5IiIiIiJyiRQ2iIjTtQzyZnT/tgDMX52E1ap9a0VERERE3JnCBhFxCeOuisTby0x61hk27T3h7HJEREREROQSKGwQEZfg5+3BuMHtAVi0/gglpRYnVyQiIiIiIvWlsEFEXMbo/m0JCfAiO7+YVduPOrscERERERGpJ4UNIuIyPD1MTLy6IwA/bErlzNlSJ1ckIiIiIiL1obBBRFzK4B6taBfqx9niMn7YmOLsckREREREpB4UNoiISzEaDUwZGQXAqu1HOZl71skViYiIiIhIXSlsEBGX07NDc3pEBmOx2li47oizyxERERERkTpS2CAiLmnyiE4YgC37Mkk+nu/sckREREREpA4UNoiIS2rfyp9BPVoBMH91IjabzckViYiIiIhIbSlsEBGXNXFYB8wmIwfSctl95LSzyxERERERkVpS2CAiLqtFoDfXDGgLwPw1SVitmt0gIiIiIuIOFDaIiEsbO7g9vs3MZJwsJHbPcWeXIyIiIiIitaCwQURcmm8zD8YOjgRg8fpkikstzi1IREREREQuSmGDiLi80f3b0DygGTkFxazclu7sckRERERE5CIUNoiIy/Mwm5g0vCMAP25KJb+oxMkViYiIiIjIhShsEBG3MLB7GBFhfpwrsfBDbIqzyxERERERkQtQ2CAibsFoMDB1ZCcAVu/MIDOnyMkViYiIiIhITRQ2iIjb6B4ZQs+OIVisNhauPeLsckREREREpAYKG0TErUwZ0QkDEHcgi6Rjec4uR0REREREqqGwQUTcSrtQP67q1QqA+auTsNlsTq5IRERERER+TWGDiLidiVd3xMNs5FB6LrsSTzu7HBERERER+RWFDSLidkICmnHtgHYAzF+TiMVqdXJFIiIiIiJyPoUNIuKWbhzUHj9vD46fLmJDwnFnlyMiIiIiIudR2CAibsmnmZmbrooEYPH6ZIpLLM4tSERERERE7BQ2iIjbGtmvDS2DmpFXWMLyuDRnlyMiIiIiIr9Q2CAibstsMnLL8CgAlm5JI6+wxMkViYiIiIgIKGwQETc3oGsoHcL9KS6x8F1ssrPLERERERERFDaIiJszGgxMGdEJgLU7j3H8dKGTKxIREREREYUNIuL2urYPpndUc6w2GwvXHnF2OSIiIiIiTZ7CBhFpFCaPiMJggO2HTpJ4NM/Z5YiIiIiINGkKG0SkUWjT0o+rY8IBmLc6EZvN5uSKRERERESaLoUNItJojB/aEU+zkcSMPHYcOuXsckREREREmiyFDSLSaAT7e3HdlREAfLs2iTKL1ckViYiIiIg0TQobRKRRuWFgBP4+HmRmF7F+1zFnlyMiIiIi0iQpbBCRRsXby8zNQzoAsGRDMmeLy5xckYiIiIhI06OwQUQaneF9WhMW7E1+USnLt6Y5uxwRERERkSZHYYOINDpmk5FbhkcBsGxrGrlnip1ckYiIiIhI06KwQUQapf7RLYlqHfD/27v7qKrqhO3j1zkHDgqIggKKmu+hmajoI06ZjtSE3TZppjjl6mVp2Z1T1mQz5aRO5nJwzXhXjj2W+YzpOFlqM+rcWuAUyRglOiSZgBQIpIBvIYKIvJ3z/OHAikCF3LDP4Xw/a7nk7N9v73OBa6tc7P3bqqp2aOenuWbHAQAAADwKZQOAdslisWjGxIGSpH99WajCs+UmJwIAAAA8B2UDgHbrxt5dNHJQNzmd0t+ScsyOAwAAAHgMygYA7dr0nw6Q1WLRoW/O6uvjJWbHAQAAADwCZQOAdq1HVz+NH95DkrT1k2w5nU6TEwEAAADtH2UDgHZvyrh+8vG26VhhqVKzzpgdBwAAAGj3KBsAtHud/X0UM6a3JOn9pBzV1DpMTgQAAAC0b5QNADxCzJgbFOBn1+lzFUpKKzQ7DgAAANCuUTYA8Agdfbw0ZVw/SdLOT3NVUVljciIAAACg/aJsAOAxbovooe5BvrpQUa0PU/LNjgMAAAC0W5QNADyGl82q6T8dIEnac+C4zpVVmpwIAAAAaJ8oGwB4lJGDumlgr86qqnFox75jZscBAAAA2iXKBgAexWKxKHbiQEnSp18V6cSZCyYnAgAAANofygYAHmdgz84aFR4sp1N6f2+O2XEAAACAdoeyAYBHum/CANmsFh3O+U6Z+efMjgMAAAC0K5QNADxS9yBfTRgRJkna9km2HE6nyYkAAACA9oOyAYDHuufWfvKx25R3skwHM0+bHQcAAABoNygbAHisAD+7/ivqBknS35JyVF3jMDkRAAAA0D5QNgDwaHf+nxvU2d+us+cv6ZNDBWbHAQAAANoFygYAHs3HbtO9t/WXJP1vcq4uXqo2OREAAADg/igbAHi8W4d1V1g3P5VfqtHu/flmxwEAAADcHmUDAI9ns1o1/acDJEn/PHhC352/ZHIiAAAAwL1RNgCApOEDuiq8dxfV1Dq0Y98xs+MAAAAAbo2yAQAkWSwWzZg4UJL02ZGT+vZUmcmJAAAAAPdF2QAA/9E/LEBjhoTIKen9vTlmxwEAAADcFmUDAHzPtPH9ZbNadCS3WOm5xWbHAQAAANwSZQMAfE9IoK8mRvaUJG37JFsOp9PkRAAAAID7oWwAgB/4+S191dHHpm9PX1BK+imz4wAAAABuh7IBAH6gk69d/zW2jyTp7//KUXVNrcmJAAAAAPfiZXaAltq/f7/efvttffnll7p48aLCwsI0adIkzZ07V76+vi061gsvvKDt27dfdc66des0fvz4JsfKy8v11ltvKSEhQYWFhfL19dXw4cM1e/ZsRUVFtSgLANfys9G9lfhFgb4rrdTHqQWaFHWD2ZEAAAAAt+FWZcOmTZu0fPlyOZ1Ode/eXT169FB2drbeeOMN7dmzR5s3b1aXLl1afNwePXqoR48eTY517ty5ye3FxcV64IEHlJubK7vdroEDB6q4uFh79+5VUlKSFi9erFmzZrU4CwDXYPe2aept/fT2B0e167M8jYvoIf+O3mbHAgAAANyC25QNR44c0e9//3tJ0ssvv6zY2FhZLBadOnVKTzzxhNLT07V48WKtXr26xce+77779NRTT7VonxdffFG5ubkaOnSo3njjDYWGhsrpdGrr1q1asmSJli9frsjISA0ZMqTFeQC4hltv7qE9B4+r4Ey5Pvg8X7HRA82OBAAAALgFt1mzYc2aNXI4HJoyZYpmzpwpi8UiSQoNDdUrr7wiq9WqPXv26OjRo62eJSMjQ4mJibJarXr11VcVGhoqSbJYLJo5c6amTJmi2tparVmzptWzAGg9VqtFM356uWD4KPW4zpZUmJwIAAAAcA9uUTaUl5dr3759kqTY2NhG43379tXYsWMlSfHx8a2eJyEhQZI0duxY9enTp9H4zJkzJUlJSUm6ePFiq+cB0HqG9Q/SkD6Bqql1avu+Y2bHAQAAANyCW5QNmZmZqqqqkt1uV0RERJNzRo0aJUn68ssvW3z8lJQUzZ8/Xw899JCefPJJvfHGGyooKLji/LS0NEnS6NGjmxyPiIiQ3W5XZWWlMjMzW5wHgOuwWCyaMXGAJOnz9FPKP1lmciIAAADA9blF2ZCbmytJCgsLk7d30wu03XDDDQ3mtsTBgweVkJCglJQU/fOf/9Rrr72mmJgYrVu3rsn5eXl5Dd7zh7y9vesXnPwxeQC4lr7dAzT2psu3S239JFtOp9PkRAAAAIBrc4sFIs+fPy/pyk+G+P5Y3dzm6NOnj1544QWNHTtWPXv2lN1uV1ZWltavX6/4+HitXLlSvr6+jZ4q0ZI8paWlzc5zJV5e1+6EbDZrg98BGGvGxIH6d9ZpZeafU+a3JYoY0PWa+3BeAq6H8xJwLZyTgOv5z/KI180tyobKykpJuuJVDZJkt9sbzG2OJ554otG24cOHa9WqVVq6dKk2b96s1157TVOnTpWfn9+PynPp0qVm52mK1WpRYKDftSf+R0BAx+t6PwBNCwz0093j+mtHUo7e35ujcZG9ZbM2729izkvA9XBeAq6FcxJof9yibPDx8ZEkVVdXX3FOVVVVg7nX69lnn9W2bdtUWlqq/fv36/bbb2+Qp6Kioll5OnTocF05HA6nSkuvvcikzWZVQEBHlZZWqLbWcV3vCaBpd47upT0p+corKtXuf2XrtuFhV53PeQm4Hs5LwLVwTgKup3PnjrJar/9qI7coG5pzi0Rzbm1oiU6dOmnQoEHKyMhQfn5+g7GAgABVVFQ0K09AQMB1Z6mpaf5fvLW1jhbNB9B8HbxtmvyTPtr2yeWrG0bdGCy7t+2a+3FeAq6H8xJwLZyTgOswankyt7g5qm/fvpKkwsLCK15N8O233zaYa4S62yRqamqazPPDEqJOdXW1CgsLDc8DwHx3jOqlrgE+OldWqY9ST5gdBwAAAHBJblE2DBkyRN7e3qqqqtLhw4ebnJOamipJGjFihCHvWVNTo2PHjkmSunfv3mCs7j3q3vOHDh8+rOrqavn4+GjIkCGG5AHgGry9bLp3fH9J0u7P81R2scrkRAAAAIDrcYuywd/fX+PGjZMkbd26tdF4Xl6e9u/fL0maNGmSIe+5ZcsWlZWVycvLS2PHjm0wFhMTI0lKSUlp8uqGLVu2SJLGjx/fYGFJAO3D2KHd1TvEXxWVtdr1WdNXOAEAAACezC3KBkmaN2+eLBaLdu7cqS1bttQ/5/706dN69tln5XA4dMcdd2jw4MEN9ouOjlZ0dLTi4+MbbE9OTtYf//hH5eXlNdheVVWlTZs2KS4uTpL0i1/8QiEhIQ3mDB06VBMnTlRtba1+9atf6fTp05Ikp9OpLVu2aOfOnbJarU0+7QKA+7NaLIqdOFCSlPjFCZ0uqTA5EQAAAOBaLE6nUcs/tL4NGzZoxYoVcjqd6tGjhwIDA5Wdna2qqir169dPmzdvVlBQUIN9wsPDJUlxcXGaNm1a/faPPvpIv/zlLyVJ3bp1U2hoqCQpNzdXFy9efvpDTEyMVq5cWf8Yy+8rLi7W/fffr7y8PNntdg0cOFDnzp1TUVGRLBaLXnzxRT344IPX/TnX1jpUXFx+zXleXlYFBvrp3LlyFtcB2sj/bElTem6xxgwJ0X9PubnROOcl4Ho4LwHXwjkJuJ6gID/ZbB7yNIo6jzzyiMLDw7V+/XodPnxY3333ncLCwjRp0iTNnTu3RbcsDB06VPPmzVNaWpry8/OVm5ur6upqBQUFady4cbr33nsVHR19xf2DgoL0t7/9TevWrVN8fLyys7Pl6+ur8ePHa86cOY1uvQDQ/sz46QBl5BbrQOZpxYwpVb8e1//0GQAAAKA9cKsrGzwRVzYAru3/7crQZ0dOKrx3F/3mgZGyWCz1Y5yXgOvhvARcC+ck4HqMurLBbdZsAABXdO9t/eVlsyrreIkO53xndhwAAADAJVA2AMB16Nq5g342upckadveHNU6+KkMAAAAQNkAANdp8k/6yK+DlwrPliv5q5NmxwEAAABMR9kAANfJt4O3fn5LX0nSjn3HVFlVa24gAAAAwGSUDQBggImRvdStcweVXKjSnn8fNzsOAAAAYCrKBgAwgLeXVdMm9Jckfbg/X6XlVSYnAgAAAMxD2QAABhkzJFR9unfSpapa/ePTXGXmFSvpixPKzCuWw8FThgEAAOA5vMwOAADthdViUezEgfrju4eUeKhAiYcK6scCO/nogTsGaVR4iIkJAQAAgLbBlQ0AYKCLl6qb3H6urFL/d/sRpWadbuNEAAAAQNujbAAAgzgcTm3+6Jurznn3o2+4pQIAAADtHmUDABjk6+MlOldWedU5xWWV+vp4SdsEAgAAAExC2QAABikpv3rR0NJ5AAAAgLuibAAAg3Tx8zF0HgAAAOCuKBsAwCA39u6iwE5XLxKCOvnoxt5d2iYQAAAAYBLKBgAwiNVq0QN3DLrqnPvvGCSr1dJGiQAAAABzUDYAgIFGhYfol/fe3OgKh6BOPvrlvTdrVHiISckAAACAtuNldgAAaG9GhYdo5KBg5RSeV7XTIm+LUwPCOnNFAwAAADwGZQMAtAKr1aIhfYMUGOinc+fKVVPjMDsSAAAA0Ga4jQIAAAAAABiKsgEAAAAAABiKsgEAAAAAABiKsgEAAAAAABiKsgEAAAAAABiKsgEAAAAAABiKsgEAAAAAABiKsgEAAAAAABiKsgEAAAAAABiKsgEAAAAAABiKsgEAAAAAABiKsgEAAAAAABiKsgEAAAAAABjK4nQ6nWaHwJU5nU45HM37I7LZrKqtdbRyIgAtwXkJuB7OS8C1cE4CrsVqtchisVz3cSgbAAAAAACAobiNAgAAAAAAGIqyAQAAAAAAGIqyAQAAAAAAGIqyAQAAAAAAGIqyAQAAAAAAGIqyAQAAAAAAGIqyAQAAAAAAGIqyAQAAAAAAGIqyAQAAAAAAGIqyAQAAAAAAGIqyAQAAAAAAGIqyAQAAAAAAGIqyAQAAAAAAGIqyAQAAAAAAGMrL7AD48c6cOaPk5GQdOXJEX331lTIzM1VZWakxY8Zo06ZNZscDPIrT6dShQ4eUmJio1NRUHTt2TBcuXFCnTp100003aerUqfr5z38ui8VidlTAo3z44Yf67LPPlJ6ertOnT6ukpETe3t7q27evJkyYoIcffliBgYFmxwQ8WlJSkubOnStJ6tmzpxITE01OBHiW1atX6/XXX7/qnJdeekn3339/i45L2eDGdu/erbi4OLNjAJC0f/9+PfLII/Wve/furZ49e6qgoEDJyclKTk7W7t27tXr1atntdvOCAh7mzTff1NGjR2W32xUcHKzw8HAVFxcrIyNDGRkZ2rp1q9avX6/BgwebHRXwSOXl5XrppZfMjgFAUteuXdWnT58mx4KDg1t8PMoGN+bv769bbrlFw4YN07Bhw5SRkaE1a9aYHQvwSE6nU7169dLDDz+syZMnq2vXrvVjO3bs0OLFi7V3716tWrVKv/71r01MCniWWbNmqV+/fhoxYoS8vb3rt2dlZem5557T119/rQULFmj37t0mpgQ816uvvqrCwkLdfvvt+vjjj82OA3i08ePHa8WKFYYdj7LBjU2fPl3Tp0+vf33q1CkT0wCeLSIiQvHx8Q2+makzdepUnTx5Uq+++qref/99LViwQFYrS+YAbSE2NrbJ7eHh4Vq+fLlmzJih7Oxs5eTkaMCAAW2cDvBsaWlpeuedd3T77bfrjjvuoGwA2hn+twsABvD392+yaKgzfvx4SVJJSYmKi4vbKhaAq+jfv3/9xxUVFSYmATxPdXW1Fi9erA4dOmjJkiVmxwHQCriyAQDawKVLl+o/7tChg4lJANRJTU2VJPn6+qpfv34mpwE8y9q1a/X1119r4cKF6t69u9lxAEg6evSoFixYoDNnzsjPz0/h4eGaPHmyBg0a9KOOR9kAAG2g7n7wwYMHy9/f3+Q0gOdyOBz1T3NauXKlJOm5556Tn5+fyckAz5GTk6O1a9dq6NChevDBB82OA+A/MjMzlZmZWf86MTFRb775ph566CE9//zzstlsLToeZQMAtLIjR47ovffek6T6R3sBaFsbNmxo9ASniIgIrVixov42JwCtz+l0atGiRaqpqdHSpUtb/M0LAOOFhIRo/vz5uu2229SrVy/5+/srNzdXmzdv1nvvvaeNGzfKy8tLv/nNb1p0XMoGAGhFZ8+e1VNPPaWamhr97Gc/0+TJk82OBHik0NBQRUZGqra2VoWFhTp79qwyMzO1c+dOjRgxQgEBAWZHBDzC5s2b9cUXX+jBBx/UsGHDzI4DQNLMmTMbbQsPD9fSpUvVq1cvrVy5Uhs3btQDDzygXr16Nfu4LBAJAK2krKxMjz32mAoLCzV06FBDHyUEoGXuuusuvfvuu9q6das+/fRT7dixQ8OHD9euXbv00EMPqba21uyIQLt36tQpvfLKKwoNDdUzzzxjdhwAzTB79myFhISopqZGiYmJLdqXsgEAWkF5ebkeffRRZWRkaNCgQfrzn//MWg2ACxk8eLDWrl2rwMBAZWZm1q+rAqD1LFu2TBcuXNCiRYv4NxFwEzabTcOHD5ck5efnt2hfbqMAAINVVFTo8ccfV1pamvr27au3335bgYGBZscC8AP+/v4aM2aMEhISlJ6ernvuucfsSEC7lpGRIUlaunSpli5d2mCs7qlNRUVFuvXWWyVJq1evVmRkZNuGBNBI3ePda2pqWrQfZQMAGKiyslJPPPGEDh48qJ49e2rDhg0KDg42OxaAK6j7jxO3UQBt5+zZs1ccczgc9ePV1dVtFQnAVXzzzTeS1OLH1FI2AIBBqqur9dRTT+nzzz9XaGioNm7cqB49epgdC8AVlJSU6MCBA5KkIUOGmJwGaP+udr/33//+dy1cuFA9e/Zs8X3hAFrP3r1768uGuquOmos1GwDAALW1tVqwYIGSkpIUHBysjRs3qnfv3mbHAjzagQMHtGbNGp04caLRWHp6uubMmaOysjKFhoZq0qRJJiQEAMBc33zzjZYsWaKjR4822O5wOLRr1y4tWLBAkjRx4kRFRES06Nhc2eDGioqKNHXq1PrXVVVVkqQvvvhCUVFR9dsfffRRPfbYY20dD/AoH374oRISEiRJdrtdv/3tb684d/HixbrpppvaKhrgsUpLS7Vq1SqtWrVKwcHBCgkJkc1mU1FRkc6cOSPp8iMx165dKz8/P5PTAgDQ9mpqarRlyxZt2bJFXbp0UVhYmGw2m7799ludP39ekjR69Gj94Q9/aPGxKRvcWG1trUpKShptr6mpabC9bsEdAK2nruyTpIKCAhUUFFxxbllZWVtEAjzeyJEjtXDhQqWkpCg7O1t5eXmqqqpSQECAoqKiFB0drenTp7MqPgDAY/Xs2VPPPPOM0tLSlJOTo/z8fFVVValz584aP3687r77bt19992y2WwtPrbF6XQ6WyEzAAAAAADwUKzZAAAAAAAADEXZAAAAAAAADEXZAAAAAAAADEXZAAAAAAAADEXZAAAAAAAADEXZAAAAAAAADEXZAAAAAAAADEXZAAAAAAAADEXZAAAAAAAADEXZAAAA8COEh4crPDxcKSkpZkcBAMDleJkdAAAAtA+rV6/W66+/3uz5WVlZrZgGAACYibIBAAAYrlu3bmZHAAAAJqJsAAAAhktOTjY7AgAAMBFrNgAAAAAAAENxZQMAADBddHS0CgoKFBcXpzvvvFNr167Vnj17VFRUpI4dO2rUqFF6/PHHNXz48Cseo7a2Vtu3b9c//vEPZWVlqby8XIGBgRo5cqRmzZqlqKioq2YoKirSpk2blJycrBMnTqi6ulohISEaNGiQYmJidNddd8nHx6fJfS9cuKB169YpISFBhYWF6tixo0aMGKF58+ZdNTMAAO0VZQMAAHAZpaWlmj59unJzc+Xt7S0fHx+VlJTo448/1ieffKJly5Zp+vTpjfYrKyvTvHnzdODAAUmSzWaTn5+fzpw5o4SEBCUkJGj27Nl6/vnnm3zfHTt2aMmSJaqsrJQkeXt7y8/PT0VFRTp+/LgSExMVHh6uIUOGNNr3zJkzmjZtmvLz8+Xj4yOr1aqSkhLt3btXycnJevPNNzVu3DgDv0oAALg+bqMAAAAu4/XXX1dxcbFee+01paWlKTU1VR988IHGjBkjh8Oh3/3ud0pPT2+034svvqgDBw7I29tbixYtUmpqqg4ePKh9+/bpvvvukyStX79e7777bqN99+7dqxdeeEGVlZWKjIzUO++8o8OHDyslJUWHDh3SO++8o9jYWHl7ezeZ+eWXX5a3t7c2btyotLQ0HTp0SNu2bVO/fv1UXV2tJUuWyOFwGPuFAgDAxVmcTqfT7BAAAMD9ff/Rl9d6GsVdd92lRYsW1b+uu41CkjZs2KCf/OQnDeZfunRJU6ZMUV5eniZMmKC33nqrfuzLL79UbGyspMvf+M+cObPR+82fP18JCQkKDAxUUlJS/e0QNTU1iomJ0YkTJzRq1Cht2LBBdru9WZ9veHi4JCkoKEi7du1S165dG4xnZWXpnnvukSRt3rxZo0aNatZxAQBoD7iyAQAAGO7s2bNX/XXhwoUm94uMjGxUNEhShw4dNGfOHEnSvn37VFZWVj/2wQcfSJK6d++uGTNmNHncp59+WpJ07ty5Bk/KSElJ0YkTJyRJCxcubHbR8H2xsbGNigbpchnRq1cvSZeLBwAAPAlrNgAAAMP92G+ux44de80xh8Oh9PT0+tdHjhyRJEVFRclqbfrnKAMGDFBoaKhOnTqlI0eOKDo6WpJ06NAhSVJwcLCGDRv2ozJfbQHIkJAQnThxQufPn/9RxwYAwF1xZQMAAHAZoaGhzRorLi6u//i777675r7S5Ssfvj9fury4oySFhYW1POx/+Pn5XXHMy+vyz3Vqamp+9PEBAHBHlA0AAMBjWSwWsyMAANAuUTYAAACXcerUqWaNBQUF1X9ct17CyZMnr3rsuvHvr69Qt5BlYWFhy8MCAIAromwAAAAuIyUl5ZpjVqtVN910U/32m2++uX78So+YzMnJqS8rvr82Q2RkpKTLt1N89dVX1xceAADUo2wAAAAuIzU1tcnCobKyUuvXr5ckjRs3TgEBAfVjkydPlnT5yodt27Y1edw//elPkqTAwEDdcsst9dujoqLUu3dvSVJcXJyqqqqM+UQAAPBwlA0AAMBldOrUSfPnz1d8fHz9ooo5OTmaO3eujh07JpvNpvnz5zfYJyIiQjExMZKkZcuW6a9//asqKiokXb5iYdGiRYqPj5d0+RGYPj4+9fvabDYtXrxYFotFqampeuSRR/Tvf/+7/gqJqqoqpaSk6LnnnlN2dnarf/4AALQXPPoSAAAY7tZbb73mnNWrV9ffxlDnySef1Hvvvaenn35adrtdPj4+Kisrk3R5MceXXnqpyUdULl++XOfOndOBAwe0bNkyxcXFyc/PT6WlpXI6nZKk2bNn6/7772+074QJE7RixQotXrxYqampmjVrlux2u3x9fXXhwoX60mPOnDkt/joAAOCpKBsAAIDhzp49e8051dXVjbYFBATo/fff19q1a7Vnzx4VFRWpS5cuGjlypB5//HGNHDmyyWN16tRJGzZs0Pbt27Vz505lZWXp4sWL6tatmyIjIzVr1ixFRUVdMcvUqVM1evRo/eUvf1FycrIKCwtVWVmpsLAw3Xjjjbrzzjs1YMCA5n8BAADwcBZnXd0PAABgkujoaBUUFCguLk7Tpk0zOw4AALhOrNkAAAAAAAAMRdkAAAAAAAAMRdkAAAAAAAAMRdkAAAAAAAAMxQKRAAAAAADAUFzZAAAAAAAADEXZAAAAAAAADEXZAAAAAAAADEXZAAAAAAAADEXZAAAAAAAADEXZAAAAAAAADEXZAAAAAAAADEXZAAAAAAAADEXZAAAAAAAADPX/AU6bJdKdDvrlAAAAAElFTkSuQmCC","text/plain":["<Figure size 1200x600 with 1 Axes>"]},"metadata":{},"output_type":"display_data"}],"source":["import pandas as pd\n","import seaborn as sns\n","import matplotlib.pyplot as plt\n","\n","df_stats = pd.read_parquet(f\"{output_dir}/training_stats.parquet\")\n","\n","# Use plot styling from seaborn.\n","sns.set(style='darkgrid')\n","\n","# Increase the plot size and font size.\n","sns.set(font_scale=1.5)\n","plt.rcParams[\"figure.figsize\"] = (12,6)\n","\n","# Plot the learning curve.\n","plt.plot(df_stats['Training Loss'], 'b-o', label=\"Training\")\n","plt.plot(df_stats['Valid. Loss'], 'g-o', label=\"Validation\")\n","\n","# Label the plot.\n","plt.title(\"Training & Validation Loss\")\n","plt.xlabel(\"Epoch\")\n","plt.ylabel(\"Loss\")\n","plt.legend()\n","plt.xticks([1, 2, 3, 4, 5])\n","\n","plt.show()"]},{"cell_type":"markdown","metadata":{"id":"dXX4z7Wqe8ry"},"source":["13. Load and test the model."]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":22809,"status":"ok","timestamp":1686768304441,"user":{"displayName":"Nicholas Corrêa","userId":"09736120585766268588"},"user_tz":-120},"id":"mpBJNtYuAY9D","outputId":"76ec7e63-ee22-45c4-b690-60ac501fbbf4"},"outputs":[{"name":"stdout","output_type":"stream","text":["Enter your question: What is the capital of France?\n","Question: 👤 What is the capital of France?\n","\n","Response 1: 🤖 The capital of France is Paris.\n","Response 2: 🤖 The capital of France is Paris.\n"]}],"source":["from transformers import AutoModelForCausalLM, AutoTokenizer\n","import torch\n","\n","device = torch.device(\"cuda\"  if torch.cuda.is_available() else  \"cpu\")\n","\n","tokenizer = AutoTokenizer.from_pretrained(output_dir)\n","aira = AutoModelForCausalLM.from_pretrained(output_dir)\n","\n","aira.eval()\n","aira.to(device)\n","\n","question =  input(\"Enter your question: \")\n","\n","inputs = tokenizer(tokenizer.bos_token + question + tokenizer.eos_token, return_tensors=\"pt\").to(device)\n","\n","responses = aira.generate(**inputs,\n","    bos_token_id=tokenizer.bos_token_id,\n","    pad_token_id=tokenizer.pad_token_id,\n","    eos_token_id=tokenizer.eos_token_id,\n","    do_sample=True,\n","    top_k=50,\n","    max_length=200,\n","    top_p=0.95,\n","    temperature=0.7,\n","    num_return_sequences=2)\n","\n","print(f\"Question: 👤 {question}\\n\")\n","\n","for i, response in  enumerate(responses):\n","    # print only the response and remove the question\n","    print(f'Response {i+1}: 🤖 {tokenizer.decode(response, skip_special_tokens=True).replace(question, \"\")}')\n"]},{"cell_type":"markdown","metadata":{"id":"cJXlHi2oe8rz"},"source":["Done! 🤗"]}],"metadata":{"accelerator":"GPU","colab":{"gpuType":"A100","machine_shape":"hm","provenance":[]},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.9.13"},"widgets":{"application/vnd.jupyter.widget-state+json":{"0008ec0128a74eaba3678973d123af9a":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"03292a675acf4db6947d6701d23fda8f":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"07673191fca947e38ee294794f81eb9f":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_7b724b134e494fd0af9359f9c1814f9f","placeholder":"​","style":"IPY_MODEL_3f4a13cbdf0f4c148c03c5ca1cf416c5","value":"Downloading (…)olve/main/merges.txt: "}},"07e0a69209424a36af75143ffe24ce24":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_85667b6eb2764e859b3e4daf67f2922f","placeholder":"​","style":"IPY_MODEL_73e56ce685b34d868e88f7feae113b73","value":" 689/689 [00:00&lt;00:00, 59.2kB/s]"}},"089a2b41007446be84c9face82cce498":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":"20px"}},"09ebe3ce4af048da8c89f33fc8735080":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"10ce7e44cdd745c9bfe19f1fecd3a327":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":"20px"}},"1157a3061d344f25ae3433fcb07e692c":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"15115eecc97c4b719f482fbd03971627":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_d76adfa0730b4e10be62eb71a169a324","IPY_MODEL_576e5200ec394e96b118e2ad8284334e","IPY_MODEL_417458df670b47478df012d9a8dd83f0"],"layout":"IPY_MODEL_48e8ea41008b402291fde670c9b0f838"}},"1802795ff27a45fb89f2f1cc9d670efa":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_657aab3bbcd9441b8d1eeed0304e86a3","max":6431878936,"min":0,"orientation":"horizontal","style":"IPY_MODEL_ce831862e4e94a8aa0918f2158d67392","value":6431878936}},"29d9a194f848426cb340ae0cc74c9a8e":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"2affe04d18994a99882b170e237bf6c6":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_10ce7e44cdd745c9bfe19f1fecd3a327","max":1,"min":0,"orientation":"horizontal","style":"IPY_MODEL_5a0cd808c80f4a189068851b9294147e","value":1}},"378582eca5544619a05a6acc62c0e7e0":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"3810e8894371460f99bae37100f6bbf3":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_5f4ba651d3ff4a6caa7c58ae6a5765d0","IPY_MODEL_4c0621b826c543a3bf4b4814d49b03ca","IPY_MODEL_69490541eed442599ce7407614edcfcb"],"layout":"IPY_MODEL_6479e02a6d3c4681848af06e62d0662e"}},"39b9b8e6871a43b3b87cb9e14672df16":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"3f4a13cbdf0f4c148c03c5ca1cf416c5":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"417458df670b47478df012d9a8dd83f0":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_f569465326a642e898847d3929daa508","placeholder":"​","style":"IPY_MODEL_d80df8cc256742df96ac16847523f11d","value":" 124/124 [00:00&lt;00:00, 10.8kB/s]"}},"432ac604f66245a7b859a76915718b99":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"48e8ea41008b402291fde670c9b0f838":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"4c0621b826c543a3bf4b4814d49b03ca":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_089a2b41007446be84c9face82cce498","max":1,"min":0,"orientation":"horizontal","style":"IPY_MODEL_c57fadd502b048dca7d250e178ebbbe1","value":1}},"56e10d229ff242c795b686e05e9350ad":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"576e5200ec394e96b118e2ad8284334e":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_bf19c1ff40d641cba4ea07e51203c6cf","max":124,"min":0,"orientation":"horizontal","style":"IPY_MODEL_c1a3598e009b421c8799860a13b320f4","value":124}},"5a0cd808c80f4a189068851b9294147e":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"5f4ba651d3ff4a6caa7c58ae6a5765d0":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_03292a675acf4db6947d6701d23fda8f","placeholder":"​","style":"IPY_MODEL_56e10d229ff242c795b686e05e9350ad","value":"Downloading (…)olve/main/vocab.json: "}},"613675b2de024ff0b16d73e2d83d3ad3":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_0008ec0128a74eaba3678973d123af9a","placeholder":"​","style":"IPY_MODEL_09ebe3ce4af048da8c89f33fc8735080","value":"Downloading pytorch_model.bin: 100%"}},"6479e02a6d3c4681848af06e62d0662e":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"657aab3bbcd9441b8d1eeed0304e86a3":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"69490541eed442599ce7407614edcfcb":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_39b9b8e6871a43b3b87cb9e14672df16","placeholder":"​","style":"IPY_MODEL_82be6e79314d42219d7823d98636c187","value":" 1.04M/? [00:00&lt;00:00, 16.6MB/s]"}},"71cca7b8799649d4a876875eda9116c0":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_29d9a194f848426cb340ae0cc74c9a8e","placeholder":"​","style":"IPY_MODEL_8f4bec13c6864463b85e5615332a165b","value":" 456k/? [00:00&lt;00:00, 11.0MB/s]"}},"73e56ce685b34d868e88f7feae113b73":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"7b724b134e494fd0af9359f9c1814f9f":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"805d2f78d9d44927bea32887f797b114":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"82be6e79314d42219d7823d98636c187":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"85667b6eb2764e859b3e4daf67f2922f":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8b01dd30e9fe4cef991299b36afb7c2f":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"8f4bec13c6864463b85e5615332a165b":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"9a3c9e1a67c1495fa09a1d2900b0dc04":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_805d2f78d9d44927bea32887f797b114","placeholder":"​","style":"IPY_MODEL_a66352b3c2d044538e3c6ed950e2cbfb","value":" 6.43G/6.43G [02:00&lt;00:00, 57.3MB/s]"}},"9c806d69e76642d0b6f023f8651ccd0f":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_613675b2de024ff0b16d73e2d83d3ad3","IPY_MODEL_1802795ff27a45fb89f2f1cc9d670efa","IPY_MODEL_9a3c9e1a67c1495fa09a1d2900b0dc04"],"layout":"IPY_MODEL_add94d96c6c24c0ca6cfc2402331b661"}},"a52ae49eabbc485dba35f93214118663":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a66352b3c2d044538e3c6ed950e2cbfb":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"add94d96c6c24c0ca6cfc2402331b661":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ae609ee3e0444ec491e080ded8e6bc0d":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ba7059e2f9b54e4c8dac069bf8bdd503":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"bf19c1ff40d641cba4ea07e51203c6cf":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c1a3598e009b421c8799860a13b320f4":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"c57fadd502b048dca7d250e178ebbbe1":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"c683803c7d6c4f8f9aef51120b6de2a4":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_dc510021f0454762a27ee6ad0ab7c843","IPY_MODEL_cf58770b9bdc42f0bc7024bb3e5b1b0a","IPY_MODEL_07e0a69209424a36af75143ffe24ce24"],"layout":"IPY_MODEL_a52ae49eabbc485dba35f93214118663"}},"ca9259567e0f4e97aac6f83026431015":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_07673191fca947e38ee294794f81eb9f","IPY_MODEL_2affe04d18994a99882b170e237bf6c6","IPY_MODEL_71cca7b8799649d4a876875eda9116c0"],"layout":"IPY_MODEL_378582eca5544619a05a6acc62c0e7e0"}},"cbf3d77427bf4760b7bbffd7f5433449":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ce831862e4e94a8aa0918f2158d67392":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"cf58770b9bdc42f0bc7024bb3e5b1b0a":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_cbf3d77427bf4760b7bbffd7f5433449","max":689,"min":0,"orientation":"horizontal","style":"IPY_MODEL_8b01dd30e9fe4cef991299b36afb7c2f","value":689}},"d76adfa0730b4e10be62eb71a169a324":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_432ac604f66245a7b859a76915718b99","placeholder":"​","style":"IPY_MODEL_ba7059e2f9b54e4c8dac069bf8bdd503","value":"Downloading (…)neration_config.json: 100%"}},"d80df8cc256742df96ac16847523f11d":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"dc510021f0454762a27ee6ad0ab7c843":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_ae609ee3e0444ec491e080ded8e6bc0d","placeholder":"​","style":"IPY_MODEL_1157a3061d344f25ae3433fcb07e692c","value":"Downloading (…)lve/main/config.json: 100%"}},"f569465326a642e898847d3929daa508":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}}}}},"nbformat":4,"nbformat_minor":0}