add noteboks

notebooks/ablation_fw_edu.ipynb

@@ -0,0 +1,2015 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Fetch the data from the hub"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 35,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import itertools\n",
+    "import pandas as pd\n",
+    "from concurrent.futures import ThreadPoolExecutor\n",
+    "from tqdm import tqdm\n",
+    "import itertools\n",
+    "import huggingface_hub\n",
+    "from tensorboard.backend.event_processing.event_accumulator import EventAccumulator\n",
+    "from huggingface_hub.utils import EntryNotFoundError"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def step_element_match(step_to_check, step_element):\n",
+    "    step_element = step_element.strip().replace(\" \", \"\")\n",
+    "    if \"-\" in step_element:\n",
+    "        a, b = step_element.split(\"-\")\n",
+    "        c = None\n",
+    "        if \"%\" in b:\n",
+    "            b, c = b.split(\"%\")\n",
+    "        return (int(a) <= step_to_check <= int(b) and\n",
+    "                (c is None or (step_to_check - int(a)) % int(c) == 0))\n",
+    "    elif \"%\" in step_element:\n",
+    "        return step_to_check % int(step_element[1:]) == 0\n",
+    "    else:\n",
+    "        return step_to_check == int(step_element)\n",
+    "    \n",
+    "def fetch_run_results_simple(repo_name, runs_to_fetch, steps_to_fetch, prefix, agg_score_columns, column_name,\n",
+    "                             seed_merge_method, oauth_token=None, prefix_file=None):\n",
+    "    if not runs_to_fetch:\n",
+    "        return\n",
+    "\n",
+    "    def fetch_run_files(run_to_fetch):\n",
+    "        def filename_to_steps_timestamp(fn):\n",
+    "            step, ts = fn.split(\"_events.out.tfevents.\")\n",
+    "            return int(step[-7:]), int(ts[:ts.index(\".\")])\n",
+    "\n",
+    "        run_to_fetch += \"_e\"\n",
+    "        try:\n",
+    "            eval_repo_file_names = [f.path for f in\n",
+    "                                    huggingface_hub.list_repo_tree(repo_name, run_to_fetch, expand=False,\n",
+    "                                                                   token=oauth_token) if\n",
+    "                                    \"_events.out.tfevents\" in f.path]\n",
+    "        except EntryNotFoundError:\n",
+    "            return []\n",
+    "\n",
+    "        eval_files = [os.path.relpath(f, run_to_fetch) for f in eval_repo_file_names]\n",
+    "        timestamps = {}\n",
+    "        for fn in eval_files:\n",
+    "            steps, ts = filename_to_steps_timestamp(fn)\n",
+    "            if steps not in timestamps or timestamps[steps][0] < ts:\n",
+    "                timestamps[steps] = ts, fn\n",
+    "\n",
+    "        results = []\n",
+    "        for eval_file, repofile in zip(eval_files, eval_repo_file_names):\n",
+    "            steps, ts = filename_to_steps_timestamp(eval_file)\n",
+    "            if not any(step_element_match(steps, step_el) for step_el in steps_to_fetch.split(\",\")):\n",
+    "                continue\n",
+    "            if timestamps[steps][1] == eval_file:\n",
+    "                results.append((run_to_fetch, steps, repofile))\n",
+    "        return results\n",
+    "\n",
+    "    def load_run_file(data):\n",
+    "        run_to_fetch, steps, repofile = data\n",
+    "        loader = EventAccumulator(huggingface_hub.hf_hub_download(repo_name, repofile, token=oauth_token))\n",
+    "        loader.Reload()\n",
+    "        runname = run_to_fetch.removeprefix(prefix).removesuffix(\"-_e\")\n",
+    "        column_names = [\"runname\", \"seed\", \"steps\", \"agg_score\"]\n",
+    "        column_values = [runname, 0, steps, 0.0]\n",
+    "\n",
+    "        for tag in loader.Tags()['scalars']:\n",
+    "            if not \"stderr\" in tag and tag.split('/')[0] == 'e':\n",
+    "                event_list = loader.Scalars(tag)\n",
+    "                tag = tag.split('/')\n",
+    "                column_names.append(f\"{tag[1]}/{tag[2]}\")\n",
+    "                column_values.append(event_list[-1].value)\n",
+    "\n",
+    "        return pd.DataFrame([column_values], columns=column_names)\n",
+    "\n",
+    "    with ThreadPoolExecutor() as pool:\n",
+    "        run_files = list(itertools.chain.from_iterable(\n",
+    "            tqdm(pool.map(fetch_run_files, runs_to_fetch), total=len(runs_to_fetch), desc=\"Fetching datafiles...\")))\n",
+    "        df = pd.concat(tqdm(pool.map(load_run_file, run_files), total=len(run_files), desc=\"Loading evals data...\"))\n",
+    "\n",
+    "    cols_to_avg = [col for col in agg_score_columns if col in df.columns]\n",
+    "    if cols_to_avg:\n",
+    "        df['agg_score'] = df[cols_to_avg].mean(axis=1)\n",
+    "\n",
+    "    prefix_file = prefix_file + \"_\" if prefix_file else \"\"\n",
+    "    df.to_csv(f\"{prefix_file}{repo_name.split('/')[-1]}_metrics.csv\", index=False)\n",
+    "    print(f\"Metrics saved to {repo_name.split('/')[-1]}_metrics.csv\")\n",
+    "\n",
+    "    return df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 37,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Fetching datafiles...: 100%|██████████| 1/1 [00:00<00:00,  1.77it/s]\n",
+      "Loading evals data...: 100%|██████████| 82/82 [00:13<00:00,  5.90it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Metrics saved to loubna-edu_fw_ablations_metrics.csv\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>runname</th>\n",
+       "      <th>seed</th>\n",
+       "      <th>steps</th>\n",
+       "      <th>agg_score</th>\n",
+       "      <th>commonsense_qa/acc</th>\n",
+       "      <th>commonsense_qa/acc_norm</th>\n",
+       "      <th>hellaswag/acc</th>\n",
+       "      <th>hellaswag/acc_norm</th>\n",
+       "      <th>openbookqa/acc</th>\n",
+       "      <th>openbookqa/acc_norm</th>\n",
+       "      <th>...</th>\n",
+       "      <th>siqa/acc</th>\n",
+       "      <th>siqa/acc_norm</th>\n",
+       "      <th>winogrande/acc</th>\n",
+       "      <th>winogrande/acc_norm</th>\n",
+       "      <th>all/acc</th>\n",
+       "      <th>all/acc_norm</th>\n",
+       "      <th>arc/acc</th>\n",
+       "      <th>arc/acc_norm</th>\n",
+       "      <th>mmlu/acc</th>\n",
+       "      <th>mmlu/acc_norm</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>edu_fineweb_350b_tokens-seed-1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>2000</td>\n",
+       "      <td>0.390326</td>\n",
+       "      <td>0.284</td>\n",
+       "      <td>0.283</td>\n",
+       "      <td>0.314</td>\n",
+       "      <td>0.325</td>\n",
+       "      <td>0.164</td>\n",
+       "      <td>0.296</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.362</td>\n",
+       "      <td>0.406</td>\n",
+       "      <td>0.511</td>\n",
+       "      <td>0.511</td>\n",
+       "      <td>0.279674</td>\n",
+       "      <td>0.299162</td>\n",
+       "      <td>0.3795</td>\n",
+       "      <td>0.3850</td>\n",
+       "      <td>0.265997</td>\n",
+       "      <td>0.284605</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>edu_fineweb_350b_tokens-seed-1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>4000</td>\n",
+       "      <td>0.414680</td>\n",
+       "      <td>0.322</td>\n",
+       "      <td>0.307</td>\n",
+       "      <td>0.343</td>\n",
+       "      <td>0.395</td>\n",
+       "      <td>0.196</td>\n",
+       "      <td>0.320</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.371</td>\n",
+       "      <td>0.388</td>\n",
+       "      <td>0.518</td>\n",
+       "      <td>0.495</td>\n",
+       "      <td>0.290613</td>\n",
+       "      <td>0.312593</td>\n",
+       "      <td>0.4215</td>\n",
+       "      <td>0.4285</td>\n",
+       "      <td>0.274401</td>\n",
+       "      <td>0.295939</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>edu_fineweb_350b_tokens-seed-1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>6000</td>\n",
+       "      <td>0.428390</td>\n",
+       "      <td>0.319</td>\n",
+       "      <td>0.311</td>\n",
+       "      <td>0.372</td>\n",
+       "      <td>0.431</td>\n",
+       "      <td>0.202</td>\n",
+       "      <td>0.352</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.373</td>\n",
+       "      <td>0.392</td>\n",
+       "      <td>0.520</td>\n",
+       "      <td>0.519</td>\n",
+       "      <td>0.303980</td>\n",
+       "      <td>0.323323</td>\n",
+       "      <td>0.4315</td>\n",
+       "      <td>0.4460</td>\n",
+       "      <td>0.288591</td>\n",
+       "      <td>0.306123</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>edu_fineweb_350b_tokens-seed-1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>8000</td>\n",
+       "      <td>0.443615</td>\n",
+       "      <td>0.340</td>\n",
+       "      <td>0.311</td>\n",
+       "      <td>0.379</td>\n",
+       "      <td>0.463</td>\n",
+       "      <td>0.204</td>\n",
+       "      <td>0.360</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.384</td>\n",
+       "      <td>0.404</td>\n",
+       "      <td>0.517</td>\n",
+       "      <td>0.517</td>\n",
+       "      <td>0.315148</td>\n",
+       "      <td>0.333284</td>\n",
+       "      <td>0.4630</td>\n",
+       "      <td>0.4790</td>\n",
+       "      <td>0.299186</td>\n",
+       "      <td>0.314921</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>edu_fineweb_350b_tokens-seed-1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>10000</td>\n",
+       "      <td>0.441457</td>\n",
+       "      <td>0.346</td>\n",
+       "      <td>0.317</td>\n",
+       "      <td>0.390</td>\n",
+       "      <td>0.454</td>\n",
+       "      <td>0.222</td>\n",
+       "      <td>0.364</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.366</td>\n",
+       "      <td>0.395</td>\n",
+       "      <td>0.514</td>\n",
+       "      <td>0.506</td>\n",
+       "      <td>0.318935</td>\n",
+       "      <td>0.335419</td>\n",
+       "      <td>0.4890</td>\n",
+       "      <td>0.4820</td>\n",
+       "      <td>0.302189</td>\n",
+       "      <td>0.317653</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>edu_fineweb_350b_tokens-seed-1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>160000</td>\n",
+       "      <td>0.507129</td>\n",
+       "      <td>0.430</td>\n",
+       "      <td>0.359</td>\n",
+       "      <td>0.473</td>\n",
+       "      <td>0.593</td>\n",
+       "      <td>0.282</td>\n",
+       "      <td>0.418</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.392</td>\n",
+       "      <td>0.402</td>\n",
+       "      <td>0.576</td>\n",
+       "      <td>0.575</td>\n",
+       "      <td>0.369137</td>\n",
+       "      <td>0.393898</td>\n",
+       "      <td>0.5670</td>\n",
+       "      <td>0.5725</td>\n",
+       "      <td>0.350226</td>\n",
+       "      <td>0.374533</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>edu_fineweb_350b_tokens-seed-1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>162000</td>\n",
+       "      <td>0.509118</td>\n",
+       "      <td>0.416</td>\n",
+       "      <td>0.367</td>\n",
+       "      <td>0.474</td>\n",
+       "      <td>0.592</td>\n",
+       "      <td>0.288</td>\n",
+       "      <td>0.408</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.390</td>\n",
+       "      <td>0.409</td>\n",
+       "      <td>0.572</td>\n",
+       "      <td>0.577</td>\n",
+       "      <td>0.367420</td>\n",
+       "      <td>0.392861</td>\n",
+       "      <td>0.5720</td>\n",
+       "      <td>0.5780</td>\n",
+       "      <td>0.348268</td>\n",
+       "      <td>0.372947</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>edu_fineweb_350b_tokens-seed-1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>164000</td>\n",
+       "      <td>0.507843</td>\n",
+       "      <td>0.416</td>\n",
+       "      <td>0.365</td>\n",
+       "      <td>0.467</td>\n",
+       "      <td>0.591</td>\n",
+       "      <td>0.276</td>\n",
+       "      <td>0.408</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.395</td>\n",
+       "      <td>0.406</td>\n",
+       "      <td>0.576</td>\n",
+       "      <td>0.580</td>\n",
+       "      <td>0.368319</td>\n",
+       "      <td>0.392000</td>\n",
+       "      <td>0.5635</td>\n",
+       "      <td>0.5715</td>\n",
+       "      <td>0.349943</td>\n",
+       "      <td>0.372246</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>edu_fineweb_350b_tokens-seed-1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>166000</td>\n",
+       "      <td>0.508308</td>\n",
+       "      <td>0.415</td>\n",
+       "      <td>0.364</td>\n",
+       "      <td>0.472</td>\n",
+       "      <td>0.593</td>\n",
+       "      <td>0.282</td>\n",
+       "      <td>0.414</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.401</td>\n",
+       "      <td>0.408</td>\n",
+       "      <td>0.575</td>\n",
+       "      <td>0.570</td>\n",
+       "      <td>0.370593</td>\n",
+       "      <td>0.393176</td>\n",
+       "      <td>0.5640</td>\n",
+       "      <td>0.5760</td>\n",
+       "      <td>0.352203</td>\n",
+       "      <td>0.373463</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>edu_fineweb_350b_tokens-seed-1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>167000</td>\n",
+       "      <td>0.509494</td>\n",
+       "      <td>0.429</td>\n",
+       "      <td>0.362</td>\n",
+       "      <td>0.472</td>\n",
+       "      <td>0.597</td>\n",
+       "      <td>0.290</td>\n",
+       "      <td>0.418</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.395</td>\n",
+       "      <td>0.404</td>\n",
+       "      <td>0.582</td>\n",
+       "      <td>0.578</td>\n",
+       "      <td>0.369666</td>\n",
+       "      <td>0.394136</td>\n",
+       "      <td>0.5670</td>\n",
+       "      <td>0.5735</td>\n",
+       "      <td>0.350671</td>\n",
+       "      <td>0.374453</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>82 rows × 22 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                           runname  seed   steps  agg_score  \\\n",
+       "0   edu_fineweb_350b_tokens-seed-1     0    2000   0.390326   \n",
+       "0   edu_fineweb_350b_tokens-seed-1     0    4000   0.414680   \n",
+       "0   edu_fineweb_350b_tokens-seed-1     0    6000   0.428390   \n",
+       "0   edu_fineweb_350b_tokens-seed-1     0    8000   0.443615   \n",
+       "0   edu_fineweb_350b_tokens-seed-1     0   10000   0.441457   \n",
+       "..                             ...   ...     ...        ...   \n",
+       "0   edu_fineweb_350b_tokens-seed-1     0  160000   0.507129   \n",
+       "0   edu_fineweb_350b_tokens-seed-1     0  162000   0.509118   \n",
+       "0   edu_fineweb_350b_tokens-seed-1     0  164000   0.507843   \n",
+       "0   edu_fineweb_350b_tokens-seed-1     0  166000   0.508308   \n",
+       "0   edu_fineweb_350b_tokens-seed-1     0  167000   0.509494   \n",
+       "\n",
+       "    commonsense_qa/acc  commonsense_qa/acc_norm  hellaswag/acc  \\\n",
+       "0                0.284                    0.283          0.314   \n",
+       "0                0.322                    0.307          0.343   \n",
+       "0                0.319                    0.311          0.372   \n",
+       "0                0.340                    0.311          0.379   \n",
+       "0                0.346                    0.317          0.390   \n",
+       "..                 ...                      ...            ...   \n",
+       "0                0.430                    0.359          0.473   \n",
+       "0                0.416                    0.367          0.474   \n",
+       "0                0.416                    0.365          0.467   \n",
+       "0                0.415                    0.364          0.472   \n",
+       "0                0.429                    0.362          0.472   \n",
+       "\n",
+       "    hellaswag/acc_norm  openbookqa/acc  openbookqa/acc_norm  ...  siqa/acc  \\\n",
+       "0                0.325           0.164                0.296  ...     0.362   \n",
+       "0                0.395           0.196                0.320  ...     0.371   \n",
+       "0                0.431           0.202                0.352  ...     0.373   \n",
+       "0                0.463           0.204                0.360  ...     0.384   \n",
+       "0                0.454           0.222                0.364  ...     0.366   \n",
+       "..                 ...             ...                  ...  ...       ...   \n",
+       "0                0.593           0.282                0.418  ...     0.392   \n",
+       "0                0.592           0.288                0.408  ...     0.390   \n",
+       "0                0.591           0.276                0.408  ...     0.395   \n",
+       "0                0.593           0.282                0.414  ...     0.401   \n",
+       "0                0.597           0.290                0.418  ...     0.395   \n",
+       "\n",
+       "    siqa/acc_norm  winogrande/acc  winogrande/acc_norm   all/acc  \\\n",
+       "0           0.406           0.511                0.511  0.279674   \n",
+       "0           0.388           0.518                0.495  0.290613   \n",
+       "0           0.392           0.520                0.519  0.303980   \n",
+       "0           0.404           0.517                0.517  0.315148   \n",
+       "0           0.395           0.514                0.506  0.318935   \n",
+       "..            ...             ...                  ...       ...   \n",
+       "0           0.402           0.576                0.575  0.369137   \n",
+       "0           0.409           0.572                0.577  0.367420   \n",
+       "0           0.406           0.576                0.580  0.368319   \n",
+       "0           0.408           0.575                0.570  0.370593   \n",
+       "0           0.404           0.582                0.578  0.369666   \n",
+       "\n",
+       "    all/acc_norm  arc/acc  arc/acc_norm  mmlu/acc  mmlu/acc_norm  \n",
+       "0       0.299162   0.3795        0.3850  0.265997       0.284605  \n",
+       "0       0.312593   0.4215        0.4285  0.274401       0.295939  \n",
+       "0       0.323323   0.4315        0.4460  0.288591       0.306123  \n",
+       "0       0.333284   0.4630        0.4790  0.299186       0.314921  \n",
+       "0       0.335419   0.4890        0.4820  0.302189       0.317653  \n",
+       "..           ...      ...           ...       ...            ...  \n",
+       "0       0.393898   0.5670        0.5725  0.350226       0.374533  \n",
+       "0       0.392861   0.5720        0.5780  0.348268       0.372947  \n",
+       "0       0.392000   0.5635        0.5715  0.349943       0.372246  \n",
+       "0       0.393176   0.5640        0.5760  0.352203       0.373463  \n",
+       "0       0.394136   0.5670        0.5735  0.350671       0.374453  \n",
+       "\n",
+       "[82 rows x 22 columns]"
+      ]
+     },
+     "execution_count": 37,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "token = os.getenv(\"HF_TOKEN\")\n",
+    "repo_name = \"HuggingFaceTB/loubna-edu_fw_ablations\"\n",
+    "runs_to_fetch = [\"tb/edu_fw_ablations-1p82G-edu_fineweb_350b_tokens-seed-1-\"]\n",
+    "steps_to_fetch = \"%1000\"\n",
+    "prefix = \"tb/edu_fw_ablations-1p82G-\"\n",
+    "metrics = ['commonsense_qa/acc_norm', 'hellaswag/acc_norm', 'openbookqa/acc_norm', 'piqa/acc_norm',\n",
+    "                   'siqa/acc_norm', 'winogrande/acc_norm', 'arc/acc_norm', 'mmlu/acc_norm']\n",
+    "agg_score_columns = metrics\n",
+    "column_name = \"agg_score\"\n",
+    "seed_merge_method = \"mean\"\n",
+    "oauth_token = token\n",
+    "\n",
+    "# runs_to_fetch = [prefix + run for run in runs_to_fetch]\n",
+    "fetch_run_results_simple(repo_name, runs_to_fetch, steps_to_fetch, prefix, agg_score_columns, column_name, seed_merge_method, oauth_token=token)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Plot the data"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Load csvs for FW and FW-Edu"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>runname</th>\n",
+       "      <th>steps</th>\n",
+       "      <th>agg_score</th>\n",
+       "      <th>commonsense_qa/acc</th>\n",
+       "      <th>commonsense_qa/acc_norm</th>\n",
+       "      <th>hellaswag/acc</th>\n",
+       "      <th>hellaswag/acc_norm</th>\n",
+       "      <th>openbookqa/acc</th>\n",
+       "      <th>openbookqa/acc_norm</th>\n",
+       "      <th>piqa/acc</th>\n",
+       "      <th>piqa/acc_norm</th>\n",
+       "      <th>siqa/acc</th>\n",
+       "      <th>siqa/acc_norm</th>\n",
+       "      <th>winogrande/acc</th>\n",
+       "      <th>winogrande/acc_norm</th>\n",
+       "      <th>arc/acc</th>\n",
+       "      <th>arc/acc_norm</th>\n",
+       "      <th>mmlu/acc</th>\n",
+       "      <th>mmlu/acc_norm</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>FineWeb-Edu</td>\n",
+       "      <td>2000</td>\n",
+       "      <td>0.390326</td>\n",
+       "      <td>0.284</td>\n",
+       "      <td>0.283</td>\n",
+       "      <td>0.314</td>\n",
+       "      <td>0.325</td>\n",
+       "      <td>0.164</td>\n",
+       "      <td>0.296</td>\n",
+       "      <td>0.623</td>\n",
+       "      <td>0.632</td>\n",
+       "      <td>0.362</td>\n",
+       "      <td>0.406</td>\n",
+       "      <td>0.511</td>\n",
+       "      <td>0.511</td>\n",
+       "      <td>0.3795</td>\n",
+       "      <td>0.3850</td>\n",
+       "      <td>0.265997</td>\n",
+       "      <td>0.284605</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>FineWeb-Edu</td>\n",
+       "      <td>4000</td>\n",
+       "      <td>0.414680</td>\n",
+       "      <td>0.322</td>\n",
+       "      <td>0.307</td>\n",
+       "      <td>0.343</td>\n",
+       "      <td>0.395</td>\n",
+       "      <td>0.196</td>\n",
+       "      <td>0.320</td>\n",
+       "      <td>0.656</td>\n",
+       "      <td>0.688</td>\n",
+       "      <td>0.371</td>\n",
+       "      <td>0.388</td>\n",
+       "      <td>0.518</td>\n",
+       "      <td>0.495</td>\n",
+       "      <td>0.4215</td>\n",
+       "      <td>0.4285</td>\n",
+       "      <td>0.274401</td>\n",
+       "      <td>0.295939</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>FineWeb-Edu</td>\n",
+       "      <td>6000</td>\n",
+       "      <td>0.428390</td>\n",
+       "      <td>0.319</td>\n",
+       "      <td>0.311</td>\n",
+       "      <td>0.372</td>\n",
+       "      <td>0.431</td>\n",
+       "      <td>0.202</td>\n",
+       "      <td>0.352</td>\n",
+       "      <td>0.660</td>\n",
+       "      <td>0.670</td>\n",
+       "      <td>0.373</td>\n",
+       "      <td>0.392</td>\n",
+       "      <td>0.520</td>\n",
+       "      <td>0.519</td>\n",
+       "      <td>0.4315</td>\n",
+       "      <td>0.4460</td>\n",
+       "      <td>0.288591</td>\n",
+       "      <td>0.306123</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>FineWeb-Edu</td>\n",
+       "      <td>8000</td>\n",
+       "      <td>0.443615</td>\n",
+       "      <td>0.340</td>\n",
+       "      <td>0.311</td>\n",
+       "      <td>0.379</td>\n",
+       "      <td>0.463</td>\n",
+       "      <td>0.204</td>\n",
+       "      <td>0.360</td>\n",
+       "      <td>0.681</td>\n",
+       "      <td>0.700</td>\n",
+       "      <td>0.384</td>\n",
+       "      <td>0.404</td>\n",
+       "      <td>0.517</td>\n",
+       "      <td>0.517</td>\n",
+       "      <td>0.4630</td>\n",
+       "      <td>0.4790</td>\n",
+       "      <td>0.299186</td>\n",
+       "      <td>0.314921</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>FineWeb-Edu</td>\n",
+       "      <td>10000</td>\n",
+       "      <td>0.441457</td>\n",
+       "      <td>0.346</td>\n",
+       "      <td>0.317</td>\n",
+       "      <td>0.390</td>\n",
+       "      <td>0.454</td>\n",
+       "      <td>0.222</td>\n",
+       "      <td>0.364</td>\n",
+       "      <td>0.690</td>\n",
+       "      <td>0.696</td>\n",
+       "      <td>0.366</td>\n",
+       "      <td>0.395</td>\n",
+       "      <td>0.514</td>\n",
+       "      <td>0.506</td>\n",
+       "      <td>0.4890</td>\n",
+       "      <td>0.4820</td>\n",
+       "      <td>0.302189</td>\n",
+       "      <td>0.317653</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "       runname  steps  agg_score  commonsense_qa/acc  commonsense_qa/acc_norm  \\\n",
+       "0  FineWeb-Edu   2000   0.390326               0.284                    0.283   \n",
+       "1  FineWeb-Edu   4000   0.414680               0.322                    0.307   \n",
+       "2  FineWeb-Edu   6000   0.428390               0.319                    0.311   \n",
+       "3  FineWeb-Edu   8000   0.443615               0.340                    0.311   \n",
+       "4  FineWeb-Edu  10000   0.441457               0.346                    0.317   \n",
+       "\n",
+       "   hellaswag/acc  hellaswag/acc_norm  openbookqa/acc  openbookqa/acc_norm  \\\n",
+       "0          0.314               0.325           0.164                0.296   \n",
+       "1          0.343               0.395           0.196                0.320   \n",
+       "2          0.372               0.431           0.202                0.352   \n",
+       "3          0.379               0.463           0.204                0.360   \n",
+       "4          0.390               0.454           0.222                0.364   \n",
+       "\n",
+       "   piqa/acc  piqa/acc_norm  siqa/acc  siqa/acc_norm  winogrande/acc  \\\n",
+       "0     0.623          0.632     0.362          0.406           0.511   \n",
+       "1     0.656          0.688     0.371          0.388           0.518   \n",
+       "2     0.660          0.670     0.373          0.392           0.520   \n",
+       "3     0.681          0.700     0.384          0.404           0.517   \n",
+       "4     0.690          0.696     0.366          0.395           0.514   \n",
+       "\n",
+       "   winogrande/acc_norm  arc/acc  arc/acc_norm  mmlu/acc  mmlu/acc_norm  \n",
+       "0                0.511   0.3795        0.3850  0.265997       0.284605  \n",
+       "1                0.495   0.4215        0.4285  0.274401       0.295939  \n",
+       "2                0.519   0.4315        0.4460  0.288591       0.306123  \n",
+       "3                0.517   0.4630        0.4790  0.299186       0.314921  \n",
+       "4                0.506   0.4890        0.4820  0.302189       0.317653  "
+      ]
+     },
+     "execution_count": 38,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "\n",
+    "# load guilherme csv with all the FW runs\n",
+    "df = pd.read_csv(\"../src_data/eval_results.csv\")\n",
+    "\n",
+    "# load FineWeb-Edu csv\n",
+    "df_2 = pd.read_csv(\"./loubna-edu_fw_ablations_metrics.csv\")\n",
+    "df_2['runname'] = df_2['runname'].replace('edu_fineweb_350b_tokens-seed-1', 'FineWeb-Edu', regex=True)\n",
+    "df_2.drop([\"seed\", \"all/acc\", \"all/acc_norm\"], axis=1, inplace=True)\n",
+    "df_2.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>runname</th>\n",
+       "      <th>steps</th>\n",
+       "      <th>agg_score</th>\n",
+       "      <th>commonsense_qa/acc</th>\n",
+       "      <th>commonsense_qa/acc_norm</th>\n",
+       "      <th>hellaswag/acc</th>\n",
+       "      <th>hellaswag/acc_norm</th>\n",
+       "      <th>openbookqa/acc</th>\n",
+       "      <th>openbookqa/acc_norm</th>\n",
+       "      <th>piqa/acc</th>\n",
+       "      <th>...</th>\n",
+       "      <th>siqa/acc</th>\n",
+       "      <th>siqa/acc_norm</th>\n",
+       "      <th>winogrande/acc</th>\n",
+       "      <th>winogrande/acc_norm</th>\n",
+       "      <th>sciq/acc</th>\n",
+       "      <th>sciq/acc_norm</th>\n",
+       "      <th>arc/acc</th>\n",
+       "      <th>arc/acc_norm</th>\n",
+       "      <th>mmlu/acc</th>\n",
+       "      <th>mmlu/acc_norm</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>1253</th>\n",
+       "      <td>FineWeb-Edu</td>\n",
+       "      <td>160000</td>\n",
+       "      <td>0.507129</td>\n",
+       "      <td>0.430</td>\n",
+       "      <td>0.359</td>\n",
+       "      <td>0.473</td>\n",
+       "      <td>0.593</td>\n",
+       "      <td>0.282</td>\n",
+       "      <td>0.418</td>\n",
+       "      <td>0.744</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.392</td>\n",
+       "      <td>0.402</td>\n",
+       "      <td>0.576</td>\n",
+       "      <td>0.575</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.5670</td>\n",
+       "      <td>0.5725</td>\n",
+       "      <td>0.350226</td>\n",
+       "      <td>0.374533</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1254</th>\n",
+       "      <td>FineWeb-Edu</td>\n",
+       "      <td>162000</td>\n",
+       "      <td>0.509118</td>\n",
+       "      <td>0.416</td>\n",
+       "      <td>0.367</td>\n",
+       "      <td>0.474</td>\n",
+       "      <td>0.592</td>\n",
+       "      <td>0.288</td>\n",
+       "      <td>0.408</td>\n",
+       "      <td>0.747</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.390</td>\n",
+       "      <td>0.409</td>\n",
+       "      <td>0.572</td>\n",
+       "      <td>0.577</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.5720</td>\n",
+       "      <td>0.5780</td>\n",
+       "      <td>0.348268</td>\n",
+       "      <td>0.372947</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1255</th>\n",
+       "      <td>FineWeb-Edu</td>\n",
+       "      <td>164000</td>\n",
+       "      <td>0.507843</td>\n",
+       "      <td>0.416</td>\n",
+       "      <td>0.365</td>\n",
+       "      <td>0.467</td>\n",
+       "      <td>0.591</td>\n",
+       "      <td>0.276</td>\n",
+       "      <td>0.408</td>\n",
+       "      <td>0.737</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.395</td>\n",
+       "      <td>0.406</td>\n",
+       "      <td>0.576</td>\n",
+       "      <td>0.580</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.5635</td>\n",
+       "      <td>0.5715</td>\n",
+       "      <td>0.349943</td>\n",
+       "      <td>0.372246</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1256</th>\n",
+       "      <td>FineWeb-Edu</td>\n",
+       "      <td>166000</td>\n",
+       "      <td>0.508308</td>\n",
+       "      <td>0.415</td>\n",
+       "      <td>0.364</td>\n",
+       "      <td>0.472</td>\n",
+       "      <td>0.593</td>\n",
+       "      <td>0.282</td>\n",
+       "      <td>0.414</td>\n",
+       "      <td>0.740</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.401</td>\n",
+       "      <td>0.408</td>\n",
+       "      <td>0.575</td>\n",
+       "      <td>0.570</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.5640</td>\n",
+       "      <td>0.5760</td>\n",
+       "      <td>0.352203</td>\n",
+       "      <td>0.373463</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1257</th>\n",
+       "      <td>FineWeb-Edu</td>\n",
+       "      <td>167000</td>\n",
+       "      <td>0.509494</td>\n",
+       "      <td>0.429</td>\n",
+       "      <td>0.362</td>\n",
+       "      <td>0.472</td>\n",
+       "      <td>0.597</td>\n",
+       "      <td>0.290</td>\n",
+       "      <td>0.418</td>\n",
+       "      <td>0.738</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.395</td>\n",
+       "      <td>0.404</td>\n",
+       "      <td>0.582</td>\n",
+       "      <td>0.578</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.5670</td>\n",
+       "      <td>0.5735</td>\n",
+       "      <td>0.350671</td>\n",
+       "      <td>0.374453</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>5 rows × 21 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "          runname   steps  agg_score  commonsense_qa/acc  \\\n",
+       "1253  FineWeb-Edu  160000   0.507129               0.430   \n",
+       "1254  FineWeb-Edu  162000   0.509118               0.416   \n",
+       "1255  FineWeb-Edu  164000   0.507843               0.416   \n",
+       "1256  FineWeb-Edu  166000   0.508308               0.415   \n",
+       "1257  FineWeb-Edu  167000   0.509494               0.429   \n",
+       "\n",
+       "      commonsense_qa/acc_norm  hellaswag/acc  hellaswag/acc_norm  \\\n",
+       "1253                    0.359          0.473               0.593   \n",
+       "1254                    0.367          0.474               0.592   \n",
+       "1255                    0.365          0.467               0.591   \n",
+       "1256                    0.364          0.472               0.593   \n",
+       "1257                    0.362          0.472               0.597   \n",
+       "\n",
+       "      openbookqa/acc  openbookqa/acc_norm  piqa/acc  ...  siqa/acc  \\\n",
+       "1253           0.282                0.418     0.744  ...     0.392   \n",
+       "1254           0.288                0.408     0.747  ...     0.390   \n",
+       "1255           0.276                0.408     0.737  ...     0.395   \n",
+       "1256           0.282                0.414     0.740  ...     0.401   \n",
+       "1257           0.290                0.418     0.738  ...     0.395   \n",
+       "\n",
+       "      siqa/acc_norm  winogrande/acc  winogrande/acc_norm  sciq/acc  \\\n",
+       "1253          0.402           0.576                0.575       NaN   \n",
+       "1254          0.409           0.572                0.577       NaN   \n",
+       "1255          0.406           0.576                0.580       NaN   \n",
+       "1256          0.408           0.575                0.570       NaN   \n",
+       "1257          0.404           0.582                0.578       NaN   \n",
+       "\n",
+       "      sciq/acc_norm  arc/acc  arc/acc_norm  mmlu/acc  mmlu/acc_norm  \n",
+       "1253            NaN   0.5670        0.5725  0.350226       0.374533  \n",
+       "1254            NaN   0.5720        0.5780  0.348268       0.372947  \n",
+       "1255            NaN   0.5635        0.5715  0.349943       0.372246  \n",
+       "1256            NaN   0.5640        0.5760  0.352203       0.373463  \n",
+       "1257            NaN   0.5670        0.5735  0.350671       0.374453  \n",
+       "\n",
+       "[5 rows x 21 columns]"
+      ]
+     },
+     "execution_count": 39,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_full = pd.concat([df, df_2], ignore_index=True)\n",
+    "df_full.tail()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Guilherme-Board plot"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 45,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "from matplotlib import pyplot as plt\n",
+    "metrics = ['agg_score', 'commonsense_qa/acc_norm', 'hellaswag/acc_norm', 'openbookqa/acc_norm', 'piqa/acc_norm',\n",
+    "                   'siqa/acc_norm', 'winogrande/acc_norm', 'arc/acc_norm', 'mmlu/acc_norm']\n",
+    "\n",
+    "def normalize_runname(runname):\n",
+    "    return runname.replace(\"/\", \"_\")\n",
+    "\n",
+    "grouped = (\n",
+    "    df_full.groupby([\"runname\", \"steps\"])\n",
+    "    .agg(\n",
+    "        {\n",
+    "            key: \"mean\" for key in metrics\n",
+    "        }\n",
+    "    )\n",
+    "    .reset_index()\n",
+    ")\n",
+    "\n",
+    "file_id=\"../assets/data/plots/edu_ablations\"\n",
+    "files = {}\n",
+    "for metric in metrics:\n",
+    "    datas = {}\n",
+    "    for name, group in grouped.groupby(\"runname\"):\n",
+    "        group = group[[\"steps\", metric]].sort_values(by=\"steps\")\n",
+    "        group = group.set_index(\"steps\")\n",
+    "        rolling_avg = group\n",
+    "        # rolling_avg = group.rolling(window=5).mean()\n",
+    "        datas[name] = {\n",
+    "            \"x\": (rolling_avg.index * 2048 * 1024 * 1e-9).tolist(),\n",
+    "            \"y\": rolling_avg[metric].tolist(),\n",
+    "            \"label\": name,\n",
+    "        }\n",
+    "    # Sort the datata based on the steps\n",
+    "    datas = {k: v for k, v in sorted(datas.items(), key=lambda x: -x[1][\"y\"][-1])}\n",
+    "    # Create a folder\n",
+    "    os.makedirs(f\"{file_id}\", exist_ok=True)\n",
+    "    with open(f\"{file_id}/{normalize_runname(metric)}.json\", \"w\") as f:\n",
+    "        json.dump({\n",
+    "            \"data\": datas,\n",
+    "            \"layout\": {\n",
+    "                \"title\": {\n",
+    "                    \"text\": \"FineWeb-Edu ablations\"\n",
+    "                },\n",
+    "            }\n",
+    "        }, f)\n",
+    "    files[metric] = {\"file\": f\"{normalize_runname(metric)}.json\"}\n",
+    "# Create index\n",
+    "with open(f\"{file_id}/index.json\", \"w\") as f:\n",
+    "    json.dump({\n",
+    "        \"files\": files,\n",
+    "        \"settings\": {\n",
+    "            \"defaultMetric\": \"agg_score\",\n",
+    "            \"slider\":{\"min\":0,\"max\":30,\"default\":5}\n",
+    "        }\n",
+    "    }, f)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Barplot"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Requirement already satisfied: kaleido in /Users/hynky/.pyenv/versions/3.12.2/envs/datatrove/lib/python3.12/site-packages (0.2.1)\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip install -U kaleido"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Plot saved to plots/edu-100k.png\n"
+     ]
+    }
+   ],
+   "source": [
+    "import plotly.express as px\n",
+    "from plotly.subplots import make_subplots\n",
+    "import plotly.graph_objects as go\n",
+    "\n",
+    "import json\n",
+    "\n",
+    "BASELINES = {\n",
+    "    \"mmlu/acc_norm\": 0.25,\n",
+    "    \"arc/acc_norm\": 0.25,\n",
+    "    \"openbookqa/acc_norm\": 0.25,\n",
+    "    \"piqa/acc_norm\": 0.5,\n",
+    "    \"hellaswag/acc_norm\": 0.25,\n",
+    "    \"siqa/acc_norm\": 0.33,\n",
+    "    \"winogrande/acc_norm\": 0.5,\n",
+    "}\n",
+    "\n",
+    "\n",
+    "def normalize_run_name(run_name):\n",
+    "    return run_name.replace(\"/\", \"_\")\n",
+    "\n",
+    "\n",
+    "def save_for_bar(dir_name, df, metrics, default_metric=\"mmlu/acc_norm\", xlabel=\"Dataset\", plot_name=\"plot name\", custom_layout={}, ranges={}):\n",
+    "    import os\n",
+    "    files = {}\n",
+    "    os.makedirs(f\"../assets/data/plots/{dir_name}\", exist_ok=True)\n",
+    "    for metric in metrics:\n",
+    "        data = {}\n",
+    "        for run_name in df[\"runname\"].unique():\n",
+    "            data[run_name] = {\n",
+    "                \"x\": [run_name],\n",
+    "                \"y\": df[df[\"runname\"] == run_name][metric].tolist(),\n",
+    "                \"label\": run_name,\n",
+    "            }\n",
+    "        file_name = f\"{normalize_run_name(metric)}.json\"\n",
+    "        files[metric] = {\"file\": f\"{file_name}\"}\n",
+    "        with open(f\"../assets/data/plots/{dir_name}/{file_name}\", \"w\") as f:\n",
+    "            json.dump({\n",
+    "                \"data\": data,\n",
+    "                \"layout\": {\n",
+    "                    \"showlegend\": False,\n",
+    "                    \"title\": {\n",
+    "                        \"text\": plot_name,\n",
+    "                    },\n",
+    "                    \"xaxis\": {\n",
+    "                        \"title\": {\n",
+    "                            \"text\": xlabel,\n",
+    "                            \"standoff\": 30\n",
+    "                        },\n",
+    "                        \"tickangle\": 30\n",
+    "                    },\n",
+    "                    \"yaxis\": {\n",
+    "                        \"range\": ranges.get(metric, [0, 1])\n",
+    "                    },\n",
+    "                    \"margin\": {\n",
+    "                        \"b\": 100\n",
+    "                    },\n",
+    "                    **custom_layout,\n",
+    "                }\n",
+    "            }, f)\n",
+    "    with open(f\"../assets/data/plots/{dir_name}/index.json\", \"w\") as f:\n",
+    "        json.dump({\n",
+    "            \"files\": files,\n",
+    "            \"settings\": {\n",
+    "                \"defaultMetric\": default_metric,\n",
+    "                \"slider\": None,\n",
+    "                \"autoSetXRange\": False,\n",
+    "                \"type\": \"bar\"\n",
+    "            }\n",
+    "        }, f)\n",
+    "    return files\n",
+    "\n",
+    "def plot_metric_comparison(df, step, metrics, plot_name, run_name_replacements=None, output_file='comparison_plot_percentages.png', default_metric=\"mmlu/acc_norm\", custom_layout={}):\n",
+    "    \"\"\"\n",
+    "    Plot a comparison of the given metrics across different runs at the specified step and save the plot.\n",
+    "    \"\"\"\n",
+    "    if run_name_replacements:\n",
+    "        df['runname'] = df['runname'].replace(run_name_replacements)\n",
+    "\n",
+    "    df_filtered = df[df['steps'] == step]\n",
+    "\n",
+    "    # Create subplots\n",
+    "\n",
+    "\n",
+    "    ranges = {}\n",
+    "    for i, metric in enumerate(metrics):\n",
+    "        yrange_start = BASELINES.get(metric, 0) * 0.9\n",
+    "        yrange_end = max(df_filtered[metric])\n",
+    "        # Adjust the end\n",
+    "        yrange_end = yrange_end + (yrange_end - yrange_start) * 0.2\n",
+    "        ranges[metric] = [yrange_start, yrange_end]\n",
+    "    \n",
+    "    file_name=f\"plots/{output_file}.png\"\n",
+    "    # fig.write_image(file_name)\n",
+    "    print(f\"Plot saved to {file_name}\")\n",
+    "\n",
+    "    save_for_bar(output_file, df_filtered, metrics, default_metric, plot_name=plot_name, custom_layout=custom_layout, ranges=ranges)\n",
+    "\n",
+    "\n",
+    "metrics = [\n",
+    "    \"mmlu/acc_norm\",\n",
+    "    \"arc/acc_norm\",\n",
+    "    \"openbookqa/acc_norm\",\n",
+    "    \"piqa/acc_norm\",\n",
+    "    \"hellaswag/acc_norm\",\n",
+    "    \"siqa/acc_norm\",\n",
+    "    \"winogrande/acc_norm\",\n",
+    "]\n",
+    "\n",
+    "plot_metric_comparison(df_full, 100000, metrics, output_file=\"edu-100k\", plot_name=\"Evaluation results at 350B tokens\", run_name_replacements={\n",
+    "    \"FineWeb (ours)\": \"FineWeb\"\n",
+    "})"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Thresholds ablation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>runname</th>\n",
+       "      <th>steps</th>\n",
+       "      <th>agg_score</th>\n",
+       "      <th>commonsense_qa/acc</th>\n",
+       "      <th>commonsense_qa/acc_norm</th>\n",
+       "      <th>hellaswag/acc</th>\n",
+       "      <th>hellaswag/acc_norm</th>\n",
+       "      <th>openbookqa/acc</th>\n",
+       "      <th>openbookqa/acc_norm</th>\n",
+       "      <th>piqa/acc</th>\n",
+       "      <th>...</th>\n",
+       "      <th>siqa/acc</th>\n",
+       "      <th>siqa/acc_norm</th>\n",
+       "      <th>winogrande/acc</th>\n",
+       "      <th>winogrande/acc_norm</th>\n",
+       "      <th>sciq/acc</th>\n",
+       "      <th>sciq/acc_norm</th>\n",
+       "      <th>arc/acc</th>\n",
+       "      <th>arc/acc_norm</th>\n",
+       "      <th>mmlu/acc</th>\n",
+       "      <th>mmlu/acc_norm</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>C4</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.330893</td>\n",
+       "      <td>0.186</td>\n",
+       "      <td>0.233</td>\n",
+       "      <td>0.272</td>\n",
+       "      <td>0.258</td>\n",
+       "      <td>0.166</td>\n",
+       "      <td>0.286</td>\n",
+       "      <td>0.542</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.367</td>\n",
+       "      <td>0.362</td>\n",
+       "      <td>0.516</td>\n",
+       "      <td>0.497</td>\n",
+       "      <td>0.208</td>\n",
+       "      <td>0.202</td>\n",
+       "      <td>0.2195</td>\n",
+       "      <td>0.2510</td>\n",
+       "      <td>0.230294</td>\n",
+       "      <td>0.250147</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>C4</td>\n",
+       "      <td>1000</td>\n",
+       "      <td>0.355112</td>\n",
+       "      <td>0.229</td>\n",
+       "      <td>0.260</td>\n",
+       "      <td>0.286</td>\n",
+       "      <td>0.288</td>\n",
+       "      <td>0.128</td>\n",
+       "      <td>0.250</td>\n",
+       "      <td>0.614</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.351</td>\n",
+       "      <td>0.404</td>\n",
+       "      <td>0.519</td>\n",
+       "      <td>0.476</td>\n",
+       "      <td>0.565</td>\n",
+       "      <td>0.518</td>\n",
+       "      <td>0.2680</td>\n",
+       "      <td>0.2935</td>\n",
+       "      <td>0.238951</td>\n",
+       "      <td>0.250399</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>C4</td>\n",
+       "      <td>2000</td>\n",
+       "      <td>0.378435</td>\n",
+       "      <td>0.268</td>\n",
+       "      <td>0.278</td>\n",
+       "      <td>0.312</td>\n",
+       "      <td>0.330</td>\n",
+       "      <td>0.122</td>\n",
+       "      <td>0.276</td>\n",
+       "      <td>0.646</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.375</td>\n",
+       "      <td>0.400</td>\n",
+       "      <td>0.509</td>\n",
+       "      <td>0.500</td>\n",
+       "      <td>0.676</td>\n",
+       "      <td>0.577</td>\n",
+       "      <td>0.3065</td>\n",
+       "      <td>0.3230</td>\n",
+       "      <td>0.247275</td>\n",
+       "      <td>0.255482</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>C4</td>\n",
+       "      <td>3000</td>\n",
+       "      <td>0.387795</td>\n",
+       "      <td>0.280</td>\n",
+       "      <td>0.295</td>\n",
+       "      <td>0.331</td>\n",
+       "      <td>0.380</td>\n",
+       "      <td>0.152</td>\n",
+       "      <td>0.274</td>\n",
+       "      <td>0.660</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.376</td>\n",
+       "      <td>0.387</td>\n",
+       "      <td>0.512</td>\n",
+       "      <td>0.496</td>\n",
+       "      <td>0.725</td>\n",
+       "      <td>0.621</td>\n",
+       "      <td>0.3175</td>\n",
+       "      <td>0.3340</td>\n",
+       "      <td>0.254534</td>\n",
+       "      <td>0.267363</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>C4</td>\n",
+       "      <td>4000</td>\n",
+       "      <td>0.399320</td>\n",
+       "      <td>0.296</td>\n",
+       "      <td>0.298</td>\n",
+       "      <td>0.351</td>\n",
+       "      <td>0.406</td>\n",
+       "      <td>0.168</td>\n",
+       "      <td>0.282</td>\n",
+       "      <td>0.676</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.382</td>\n",
+       "      <td>0.404</td>\n",
+       "      <td>0.522</td>\n",
+       "      <td>0.503</td>\n",
+       "      <td>0.723</td>\n",
+       "      <td>0.618</td>\n",
+       "      <td>0.3255</td>\n",
+       "      <td>0.3470</td>\n",
+       "      <td>0.254762</td>\n",
+       "      <td>0.263563</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1171</th>\n",
+       "      <td>The Pile</td>\n",
+       "      <td>163000</td>\n",
+       "      <td>0.463789</td>\n",
+       "      <td>0.379</td>\n",
+       "      <td>0.349</td>\n",
+       "      <td>0.441</td>\n",
+       "      <td>0.555</td>\n",
+       "      <td>0.240</td>\n",
+       "      <td>0.366</td>\n",
+       "      <td>0.701</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.405</td>\n",
+       "      <td>0.388</td>\n",
+       "      <td>0.585</td>\n",
+       "      <td>0.560</td>\n",
+       "      <td>0.875</td>\n",
+       "      <td>0.820</td>\n",
+       "      <td>0.4475</td>\n",
+       "      <td>0.4450</td>\n",
+       "      <td>0.299378</td>\n",
+       "      <td>0.326313</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1172</th>\n",
+       "      <td>The Pile</td>\n",
+       "      <td>164000</td>\n",
+       "      <td>0.462758</td>\n",
+       "      <td>0.369</td>\n",
+       "      <td>0.344</td>\n",
+       "      <td>0.438</td>\n",
+       "      <td>0.552</td>\n",
+       "      <td>0.248</td>\n",
+       "      <td>0.348</td>\n",
+       "      <td>0.708</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.395</td>\n",
+       "      <td>0.401</td>\n",
+       "      <td>0.577</td>\n",
+       "      <td>0.567</td>\n",
+       "      <td>0.874</td>\n",
+       "      <td>0.806</td>\n",
+       "      <td>0.4465</td>\n",
+       "      <td>0.4355</td>\n",
+       "      <td>0.302083</td>\n",
+       "      <td>0.331563</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1173</th>\n",
+       "      <td>The Pile</td>\n",
+       "      <td>165000</td>\n",
+       "      <td>0.465026</td>\n",
+       "      <td>0.383</td>\n",
+       "      <td>0.350</td>\n",
+       "      <td>0.438</td>\n",
+       "      <td>0.553</td>\n",
+       "      <td>0.234</td>\n",
+       "      <td>0.352</td>\n",
+       "      <td>0.707</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.400</td>\n",
+       "      <td>0.401</td>\n",
+       "      <td>0.569</td>\n",
+       "      <td>0.556</td>\n",
+       "      <td>0.874</td>\n",
+       "      <td>0.811</td>\n",
+       "      <td>0.4460</td>\n",
+       "      <td>0.4455</td>\n",
+       "      <td>0.305193</td>\n",
+       "      <td>0.331708</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1174</th>\n",
+       "      <td>The Pile</td>\n",
+       "      <td>166000</td>\n",
+       "      <td>0.462349</td>\n",
+       "      <td>0.377</td>\n",
+       "      <td>0.346</td>\n",
+       "      <td>0.440</td>\n",
+       "      <td>0.557</td>\n",
+       "      <td>0.228</td>\n",
+       "      <td>0.346</td>\n",
+       "      <td>0.711</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.398</td>\n",
+       "      <td>0.398</td>\n",
+       "      <td>0.572</td>\n",
+       "      <td>0.558</td>\n",
+       "      <td>0.877</td>\n",
+       "      <td>0.811</td>\n",
+       "      <td>0.4525</td>\n",
+       "      <td>0.4385</td>\n",
+       "      <td>0.301952</td>\n",
+       "      <td>0.331295</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1175</th>\n",
+       "      <td>The Pile</td>\n",
+       "      <td>167000</td>\n",
+       "      <td>0.464539</td>\n",
+       "      <td>0.386</td>\n",
+       "      <td>0.354</td>\n",
+       "      <td>0.434</td>\n",
+       "      <td>0.557</td>\n",
+       "      <td>0.232</td>\n",
+       "      <td>0.356</td>\n",
+       "      <td>0.706</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.402</td>\n",
+       "      <td>0.402</td>\n",
+       "      <td>0.573</td>\n",
+       "      <td>0.559</td>\n",
+       "      <td>0.867</td>\n",
+       "      <td>0.802</td>\n",
+       "      <td>0.4475</td>\n",
+       "      <td>0.4375</td>\n",
+       "      <td>0.301934</td>\n",
+       "      <td>0.330810</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>1176 rows × 21 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "       runname   steps  agg_score  commonsense_qa/acc  \\\n",
+       "0           C4       0   0.330893               0.186   \n",
+       "1           C4    1000   0.355112               0.229   \n",
+       "2           C4    2000   0.378435               0.268   \n",
+       "3           C4    3000   0.387795               0.280   \n",
+       "4           C4    4000   0.399320               0.296   \n",
+       "...        ...     ...        ...                 ...   \n",
+       "1171  The Pile  163000   0.463789               0.379   \n",
+       "1172  The Pile  164000   0.462758               0.369   \n",
+       "1173  The Pile  165000   0.465026               0.383   \n",
+       "1174  The Pile  166000   0.462349               0.377   \n",
+       "1175  The Pile  167000   0.464539               0.386   \n",
+       "\n",
+       "      commonsense_qa/acc_norm  hellaswag/acc  hellaswag/acc_norm  \\\n",
+       "0                       0.233          0.272               0.258   \n",
+       "1                       0.260          0.286               0.288   \n",
+       "2                       0.278          0.312               0.330   \n",
+       "3                       0.295          0.331               0.380   \n",
+       "4                       0.298          0.351               0.406   \n",
+       "...                       ...            ...                 ...   \n",
+       "1171                    0.349          0.441               0.555   \n",
+       "1172                    0.344          0.438               0.552   \n",
+       "1173                    0.350          0.438               0.553   \n",
+       "1174                    0.346          0.440               0.557   \n",
+       "1175                    0.354          0.434               0.557   \n",
+       "\n",
+       "      openbookqa/acc  openbookqa/acc_norm  piqa/acc  ...  siqa/acc  \\\n",
+       "0              0.166                0.286     0.542  ...     0.367   \n",
+       "1              0.128                0.250     0.614  ...     0.351   \n",
+       "2              0.122                0.276     0.646  ...     0.375   \n",
+       "3              0.152                0.274     0.660  ...     0.376   \n",
+       "4              0.168                0.282     0.676  ...     0.382   \n",
+       "...              ...                  ...       ...  ...       ...   \n",
+       "1171           0.240                0.366     0.701  ...     0.405   \n",
+       "1172           0.248                0.348     0.708  ...     0.395   \n",
+       "1173           0.234                0.352     0.707  ...     0.400   \n",
+       "1174           0.228                0.346     0.711  ...     0.398   \n",
+       "1175           0.232                0.356     0.706  ...     0.402   \n",
+       "\n",
+       "      siqa/acc_norm  winogrande/acc  winogrande/acc_norm  sciq/acc  \\\n",
+       "0             0.362           0.516                0.497     0.208   \n",
+       "1             0.404           0.519                0.476     0.565   \n",
+       "2             0.400           0.509                0.500     0.676   \n",
+       "3             0.387           0.512                0.496     0.725   \n",
+       "4             0.404           0.522                0.503     0.723   \n",
+       "...             ...             ...                  ...       ...   \n",
+       "1171          0.388           0.585                0.560     0.875   \n",
+       "1172          0.401           0.577                0.567     0.874   \n",
+       "1173          0.401           0.569                0.556     0.874   \n",
+       "1174          0.398           0.572                0.558     0.877   \n",
+       "1175          0.402           0.573                0.559     0.867   \n",
+       "\n",
+       "      sciq/acc_norm  arc/acc  arc/acc_norm  mmlu/acc  mmlu/acc_norm  \n",
+       "0             0.202   0.2195        0.2510  0.230294       0.250147  \n",
+       "1             0.518   0.2680        0.2935  0.238951       0.250399  \n",
+       "2             0.577   0.3065        0.3230  0.247275       0.255482  \n",
+       "3             0.621   0.3175        0.3340  0.254534       0.267363  \n",
+       "4             0.618   0.3255        0.3470  0.254762       0.263563  \n",
+       "...             ...      ...           ...       ...            ...  \n",
+       "1171          0.820   0.4475        0.4450  0.299378       0.326313  \n",
+       "1172          0.806   0.4465        0.4355  0.302083       0.331563  \n",
+       "1173          0.811   0.4460        0.4455  0.305193       0.331708  \n",
+       "1174          0.811   0.4525        0.4385  0.301952       0.331295  \n",
+       "1175          0.802   0.4475        0.4375  0.301934       0.330810  \n",
+       "\n",
+       "[1176 rows x 21 columns]"
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Fetching datafiles...: 100%|██████████| 4/4 [00:00<00:00, 21.68it/s]\n",
+      "Loading evals data...: 100%|██████████| 26/26 [00:04<00:00,  5.76it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Metrics saved to loubna-ablations_faq_metrics.csv\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "token = os.getenv(\"HF_TOKEN\")\n",
+    "repo_name = \"HuggingFaceTB/loubna-ablations_faq\"\n",
+    "runs_to_fetch = [\"filtered_web_min_score_4_fix-seed-1-\", \"fineweb_2B_educational_minimum_score_3-seed-0-\", \"fineweb_2B_educational_regression-seed-6-\", \"fineweb_2024_10_all_2B-seed-6-\"]\n",
+    "steps_to_fetch = \"%1000\"\n",
+    "prefix = \"tb/ablations_faq-1p81G-\"\n",
+    "metrics = ['commonsense_qa/acc_norm', 'hellaswag/acc_norm', 'openbookqa/acc_norm', 'piqa/acc_norm',\n",
+    "                   'siqa/acc_norm', 'winogrande/acc_norm', 'arc/acc_norm', 'mmlu/acc_norm']\n",
+    "agg_score_columns = metrics\n",
+    "column_name = \"agg_score\"\n",
+    "seed_merge_method = \"mean\"\n",
+    "oauth_token = token\n",
+    "\n",
+    "runs_to_fetch = [prefix + run for run in runs_to_fetch]\n",
+    "df = fetch_run_results_simple(repo_name, runs_to_fetch, steps_to_fetch, prefix, agg_score_columns, column_name, seed_merge_method, oauth_token=token)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>runname</th>\n",
+       "      <th>seed</th>\n",
+       "      <th>steps</th>\n",
+       "      <th>agg_score</th>\n",
+       "      <th>commonsense_qa/acc</th>\n",
+       "      <th>commonsense_qa/acc_norm</th>\n",
+       "      <th>hellaswag/acc</th>\n",
+       "      <th>hellaswag/acc_norm</th>\n",
+       "      <th>openbookqa/acc</th>\n",
+       "      <th>openbookqa/acc_norm</th>\n",
+       "      <th>...</th>\n",
+       "      <th>siqa/acc</th>\n",
+       "      <th>siqa/acc_norm</th>\n",
+       "      <th>winogrande/acc</th>\n",
+       "      <th>winogrande/acc_norm</th>\n",
+       "      <th>all/acc</th>\n",
+       "      <th>all/acc_norm</th>\n",
+       "      <th>arc/acc</th>\n",
+       "      <th>arc/acc_norm</th>\n",
+       "      <th>mmlu/acc</th>\n",
+       "      <th>mmlu/acc_norm</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>FineWeb (FW)</td>\n",
+       "      <td>0</td>\n",
+       "      <td>4000</td>\n",
+       "      <td>0.389983</td>\n",
+       "      <td>0.275</td>\n",
+       "      <td>0.281</td>\n",
+       "      <td>0.352</td>\n",
+       "      <td>0.383</td>\n",
+       "      <td>0.152</td>\n",
+       "      <td>0.286</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.365</td>\n",
+       "      <td>0.385</td>\n",
+       "      <td>0.505</td>\n",
+       "      <td>0.493</td>\n",
+       "      <td>0.265054</td>\n",
+       "      <td>0.281046</td>\n",
+       "      <td>0.3265</td>\n",
+       "      <td>0.3435</td>\n",
+       "      <td>0.250500</td>\n",
+       "      <td>0.264368</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>FineWeb (FW)</td>\n",
+       "      <td>0</td>\n",
+       "      <td>5000</td>\n",
+       "      <td>0.397987</td>\n",
+       "      <td>0.303</td>\n",
+       "      <td>0.297</td>\n",
+       "      <td>0.349</td>\n",
+       "      <td>0.397</td>\n",
+       "      <td>0.154</td>\n",
+       "      <td>0.290</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.375</td>\n",
+       "      <td>0.383</td>\n",
+       "      <td>0.509</td>\n",
+       "      <td>0.502</td>\n",
+       "      <td>0.268548</td>\n",
+       "      <td>0.282678</td>\n",
+       "      <td>0.3340</td>\n",
+       "      <td>0.3560</td>\n",
+       "      <td>0.253134</td>\n",
+       "      <td>0.264896</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>FineWeb (FW)</td>\n",
+       "      <td>0</td>\n",
+       "      <td>6000</td>\n",
+       "      <td>0.403954</td>\n",
+       "      <td>0.317</td>\n",
+       "      <td>0.319</td>\n",
+       "      <td>0.359</td>\n",
+       "      <td>0.416</td>\n",
+       "      <td>0.166</td>\n",
+       "      <td>0.284</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.379</td>\n",
+       "      <td>0.400</td>\n",
+       "      <td>0.516</td>\n",
+       "      <td>0.490</td>\n",
+       "      <td>0.268197</td>\n",
+       "      <td>0.286678</td>\n",
+       "      <td>0.3330</td>\n",
+       "      <td>0.3590</td>\n",
+       "      <td>0.252102</td>\n",
+       "      <td>0.268633</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>FineWeb (FW)</td>\n",
+       "      <td>0</td>\n",
+       "      <td>7000</td>\n",
+       "      <td>0.404859</td>\n",
+       "      <td>0.298</td>\n",
+       "      <td>0.310</td>\n",
+       "      <td>0.367</td>\n",
+       "      <td>0.424</td>\n",
+       "      <td>0.176</td>\n",
+       "      <td>0.290</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.382</td>\n",
+       "      <td>0.396</td>\n",
+       "      <td>0.511</td>\n",
+       "      <td>0.494</td>\n",
+       "      <td>0.271701</td>\n",
+       "      <td>0.289459</td>\n",
+       "      <td>0.3250</td>\n",
+       "      <td>0.3510</td>\n",
+       "      <td>0.256203</td>\n",
+       "      <td>0.271874</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>FineWeb (FW)</td>\n",
+       "      <td>0</td>\n",
+       "      <td>8000</td>\n",
+       "      <td>0.403283</td>\n",
+       "      <td>0.330</td>\n",
+       "      <td>0.319</td>\n",
+       "      <td>0.364</td>\n",
+       "      <td>0.412</td>\n",
+       "      <td>0.176</td>\n",
+       "      <td>0.276</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.383</td>\n",
+       "      <td>0.403</td>\n",
+       "      <td>0.510</td>\n",
+       "      <td>0.493</td>\n",
+       "      <td>0.267533</td>\n",
+       "      <td>0.287018</td>\n",
+       "      <td>0.3295</td>\n",
+       "      <td>0.3510</td>\n",
+       "      <td>0.251046</td>\n",
+       "      <td>0.269266</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>5 rows × 22 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "        runname  seed  steps  agg_score  commonsense_qa/acc  \\\n",
+       "0  FineWeb (FW)     0   4000   0.389983               0.275   \n",
+       "0  FineWeb (FW)     0   5000   0.397987               0.303   \n",
+       "0  FineWeb (FW)     0   6000   0.403954               0.317   \n",
+       "0  FineWeb (FW)     0   7000   0.404859               0.298   \n",
+       "0  FineWeb (FW)     0   8000   0.403283               0.330   \n",
+       "\n",
+       "   commonsense_qa/acc_norm  hellaswag/acc  hellaswag/acc_norm  openbookqa/acc  \\\n",
+       "0                    0.281          0.352               0.383           0.152   \n",
+       "0                    0.297          0.349               0.397           0.154   \n",
+       "0                    0.319          0.359               0.416           0.166   \n",
+       "0                    0.310          0.367               0.424           0.176   \n",
+       "0                    0.319          0.364               0.412           0.176   \n",
+       "\n",
+       "   openbookqa/acc_norm  ...  siqa/acc  siqa/acc_norm  winogrande/acc  \\\n",
+       "0                0.286  ...     0.365          0.385           0.505   \n",
+       "0                0.290  ...     0.375          0.383           0.509   \n",
+       "0                0.284  ...     0.379          0.400           0.516   \n",
+       "0                0.290  ...     0.382          0.396           0.511   \n",
+       "0                0.276  ...     0.383          0.403           0.510   \n",
+       "\n",
+       "   winogrande/acc_norm   all/acc  all/acc_norm  arc/acc  arc/acc_norm  \\\n",
+       "0                0.493  0.265054      0.281046   0.3265        0.3435   \n",
+       "0                0.502  0.268548      0.282678   0.3340        0.3560   \n",
+       "0                0.490  0.268197      0.286678   0.3330        0.3590   \n",
+       "0                0.494  0.271701      0.289459   0.3250        0.3510   \n",
+       "0                0.493  0.267533      0.287018   0.3295        0.3510   \n",
+       "\n",
+       "   mmlu/acc  mmlu/acc_norm  \n",
+       "0  0.250500       0.264368  \n",
+       "0  0.253134       0.264896  \n",
+       "0  0.252102       0.268633  \n",
+       "0  0.256203       0.271874  \n",
+       "0  0.251046       0.269266  \n",
+       "\n",
+       "[5 rows x 22 columns]"
+      ]
+     },
+     "execution_count": 25,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df['runname'] = df['runname'].replace({\"filtered_web_min_score_4_fix-seed-1\": \"FW-Edu-threshold=4\",\n",
+    "                                        \"fineweb_2B_educational_minimum_score_3-seed-0\": \"FW-Edu-threshold=3\",\n",
+    "                                        \"fineweb_2B_educational_regression-seed-6\": \"FW-Edu-threshold=2\",\n",
+    "                                        \"fineweb_2024_10_all_2B-seed-6\": \"FineWeb (FW)\"}, regex=True)\n",
+    "df.tail()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0    FW-Edu-threshold=4\n",
+       "0    FW-Edu-threshold=4\n",
+       "0    FW-Edu-threshold=4\n",
+       "0    FW-Edu-threshold=4\n",
+       "0    FW-Edu-threshold=4\n",
+       "0    FW-Edu-threshold=4\n",
+       "0    FW-Edu-threshold=4\n",
+       "0    FW-Edu-threshold=4\n",
+       "0    FW-Edu-threshold=3\n",
+       "0    FW-Edu-threshold=3\n",
+       "0    FW-Edu-threshold=3\n",
+       "0    FW-Edu-threshold=3\n",
+       "0    FW-Edu-threshold=3\n",
+       "0    FW-Edu-threshold=2\n",
+       "0    FW-Edu-threshold=2\n",
+       "0    FW-Edu-threshold=2\n",
+       "0    FW-Edu-threshold=2\n",
+       "0    FW-Edu-threshold=2\n",
+       "0          FineWeb (FW)\n",
+       "0          FineWeb (FW)\n",
+       "0          FineWeb (FW)\n",
+       "0          FineWeb (FW)\n",
+       "0          FineWeb (FW)\n",
+       "0          FineWeb (FW)\n",
+       "0          FineWeb (FW)\n",
+       "0          FineWeb (FW)\n",
+       "Name: runname, dtype: object"
+      ]
+     },
+     "execution_count": 26,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df[\"runname\"]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Plot saved to plots/edu-8k.png\n"
+     ]
+    }
+   ],
+   "source": [
+    "\n",
+    "metrics = [\n",
+    "    \"mmlu/acc_norm\",\n",
+    "    \"arc/acc_norm\",\n",
+    "    \"openbookqa/acc_norm\",\n",
+    "    \"piqa/acc_norm\",\n",
+    "    \"hellaswag/acc_norm\",\n",
+    "    \"siqa/acc_norm\",\n",
+    "    \"winogrande/acc_norm\",\n",
+    "]\n",
+    "plot_metric_comparison(df, 8000, metrics, output_file=\"edu-8k\", plot_name=\"FineWeb-Edu thresholding\", custom_layout={\n",
+    "    \"xaxis\": {\n",
+    "        \"title\": {\n",
+    "            \"standoff\": 60,\n",
+    "            \"text\": \"Dataset\"\n",
+    "        },\n",
+    "        \"tickangle\": 30\n",
+    "    },\n",
+    "    \"margin\": {\n",
+    "        \"b\": 120\n",
+    "    }\n",
+    "})"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "textbooks",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

notebooks/check_decontamination.ipynb

@@ -0,0 +1,77 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "id": "initial_id",
+   "metadata": {
+    "collapsed": true,
+    "ExecuteTime": {
+     "end_time": "2024-05-15T07:49:59.747703Z",
+     "start_time": "2024-05-15T07:49:59.134058Z"
+    }
+   },
+   "source": [
+    "import pandas as pd\n",
+    "\n",
+    "df = pd.read_csv('/home/gui/hf_dev/datatrove/blogpost/data/decont_ngrams-per_dump.csv')"
+   ],
+   "execution_count": 1,
+   "outputs": []
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-15T07:51:52.324884Z",
+     "start_time": "2024-05-15T07:51:52.283371Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "df = df.groupby([\"ngram\", \"task\"], as_index=False)[\"count\"].sum().sort_values(\"count\", ascending=False)",
+   "id": "c691b2709c417bf4",
+   "execution_count": 8,
+   "outputs": []
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-15T07:52:17.954219Z",
+     "start_time": "2024-05-15T07:52:17.938060Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "df.to_csv('/home/gui/hf_dev/datatrove/blogpost/data/decont_ngrams-global.csv', index=False)",
+   "id": "9c0dfcd486f8e260",
+   "execution_count": 9,
+   "outputs": []
+  },
+  {
+   "metadata": {},
+   "cell_type": "code",
+   "execution_count": null,
+   "source": "",
+   "id": "d5fef0e4bc91a43e",
+   "outputs": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/check_top_60k_change.ipynb

@@ -0,0 +1,1424 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "id": "initial_id",
+   "metadata": {
+    "collapsed": true,
+    "ExecuteTime": {
+     "end_time": "2024-05-14T15:00:26.015772Z",
+     "start_time": "2024-05-14T15:00:25.963139Z"
+    }
+   },
+   "source": [
+    "import pandas as pd\n",
+    "\n",
+    "df_43 = pd.read_csv('/home/gui/hf_dev/datatrove/blogpost/data/top_60k_urls_CC_MAIN_2021_43.csv').sort_values(by=\"Frequency\", ascending=False)\n",
+    "df_49 = pd.read_csv('/home/gui/hf_dev/datatrove/blogpost/data/top_60k_urls_CC_MAIN_2021_49.csv').sort_values(by=\"Frequency\", ascending=False)"
+   ],
+   "outputs": [],
+   "execution_count": 27
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-14T15:00:27.625974Z",
+     "start_time": "2024-05-14T15:00:26.358729Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "freqs_49 = {row[1][\"URL\"]: row[1][\"Frequency\"] for row in df_49.iterrows()}",
+   "id": "6a21a1ed442a6d79",
+   "outputs": [],
+   "execution_count": 28
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-14T15:00:28.029676Z",
+     "start_time": "2024-05-14T15:00:27.626997Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "df_43['in_49'] = df_43.apply(lambda row: freqs_49.get(row[\"URL\"], 0), axis=1)\n",
+    "df_43['change_to_49'] = df_43.apply(lambda row: freqs_49.get(row[\"URL\"], 0) - row[\"Frequency\"], axis=1)"
+   ],
+   "id": "bc7cdf0d04ff0d0",
+   "outputs": [],
+   "execution_count": 29
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-14T15:00:28.035756Z",
+     "start_time": "2024-05-14T15:00:28.030629Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "df_43",
+   "id": "990f471d499d7064",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "                           URL  Frequency     in_49  change_to_49\n",
+       "0         worldwidescience.org   0.001337  0.001317 -2.004393e-05\n",
+       "1                    issuu.com   0.001006  0.000992 -1.395550e-05\n",
+       "2             en.wikipedia.org   0.000824  0.001125  3.004770e-04\n",
+       "3          caselaw.findlaw.com   0.000661  0.000228 -4.330254e-04\n",
+       "4          www.frontiersin.org   0.000611  0.000402 -2.088168e-04\n",
+       "...                        ...        ...       ...           ...\n",
+       "59995  www.basketballghana.com   0.000001  0.000002  7.485694e-07\n",
+       "59996           meisendorf.com   0.000001  0.000000 -1.323341e-06\n",
+       "59997     www.anyrubbish.co.uk   0.000001  0.000000 -1.323290e-06\n",
+       "59998               qjshhxx.cn   0.000001  0.000000 -1.323239e-06\n",
+       "59999    www.al-enterprise.com   0.000001  0.000000 -1.323225e-06\n",
+       "\n",
+       "[60000 rows x 4 columns]"
+      ],
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>URL</th>\n",
+       "      <th>Frequency</th>\n",
+       "      <th>in_49</th>\n",
+       "      <th>change_to_49</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>worldwidescience.org</td>\n",
+       "      <td>0.001337</td>\n",
+       "      <td>0.001317</td>\n",
+       "      <td>-2.004393e-05</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>issuu.com</td>\n",
+       "      <td>0.001006</td>\n",
+       "      <td>0.000992</td>\n",
+       "      <td>-1.395550e-05</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>en.wikipedia.org</td>\n",
+       "      <td>0.000824</td>\n",
+       "      <td>0.001125</td>\n",
+       "      <td>3.004770e-04</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>caselaw.findlaw.com</td>\n",
+       "      <td>0.000661</td>\n",
+       "      <td>0.000228</td>\n",
+       "      <td>-4.330254e-04</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>www.frontiersin.org</td>\n",
+       "      <td>0.000611</td>\n",
+       "      <td>0.000402</td>\n",
+       "      <td>-2.088168e-04</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>59995</th>\n",
+       "      <td>www.basketballghana.com</td>\n",
+       "      <td>0.000001</td>\n",
+       "      <td>0.000002</td>\n",
+       "      <td>7.485694e-07</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>59996</th>\n",
+       "      <td>meisendorf.com</td>\n",
+       "      <td>0.000001</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>-1.323341e-06</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>59997</th>\n",
+       "      <td>www.anyrubbish.co.uk</td>\n",
+       "      <td>0.000001</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>-1.323290e-06</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>59998</th>\n",
+       "      <td>qjshhxx.cn</td>\n",
+       "      <td>0.000001</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>-1.323239e-06</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>59999</th>\n",
+       "      <td>www.al-enterprise.com</td>\n",
+       "      <td>0.000001</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>-1.323225e-06</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>60000 rows × 4 columns</p>\n",
+       "</div>"
+      ]
+     },
+     "execution_count": 30,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 30
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-14T14:59:00.727813Z",
+     "start_time": "2024-05-14T14:59:00.713582Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "freqs_49",
+   "id": "6d9dec8ef32e61b2",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'worldwidescience.org': 0.0013371601446593,\n",
+       " 'issuu.com': 0.0010055409292792,\n",
+       " 'en.wikipedia.org': 0.0008242551989742,\n",
+       " 'caselaw.findlaw.com': 0.0006608764065299,\n",
+       " 'www.frontiersin.org': 0.0006109349846815,\n",
+       " 'chroniclingamerica.loc.gov': 0.0005375644622649,\n",
+       " 'www.nature.com': 0.0005266164399221,\n",
+       " 'lookformedical.com': 0.0004704824927555,\n",
+       " 'en.m.wikipedia.org': 0.0004443793906608,\n",
+       " 'ufdc.ufl.edu': 0.0004426934126914,\n",
+       " 'journals.plos.org': 0.0004371460070378,\n",
+       " 'casetext.com': 0.0003903933122818,\n",
+       " 'hansard.parliament.uk': 0.0003611472676514,\n",
+       " 'search.audioburst.com': 0.0003535316041847,\n",
+       " 'ivy.fm': 0.0003530053155805,\n",
+       " 'link.springer.com': 0.0003286474058102,\n",
+       " 'www.newageislam.com': 0.0003196897846979,\n",
+       " 'www.studylight.org': 0.0003159274076666,\n",
+       " 'panewsarchive.psu.edu': 0.0003121250457216,\n",
+       " 'www.theguardian.com': 0.0003038877330206,\n",
+       " 'bmcgenomics.biomedcentral.com': 0.0002553320996198,\n",
+       " 'law.justia.com': 0.0002463165002013,\n",
+       " 'www.hotfreebooks.com': 0.0002438439057772,\n",
+       " 'www.hindawi.com': 0.0002374853040569,\n",
+       " 'zims-en.kiwix.campusafrica.gos.orange.com': 0.0002318541153106,\n",
+       " 'discover.hubpages.com': 0.0002314644626707,\n",
+       " 'www.fanfiction.net': 0.00023022444825,\n",
+       " 'www.huffpost.com': 0.0002272851203057,\n",
+       " 'nebnewspapers.unl.edu': 0.0002254127055625,\n",
+       " 'www.angi.com': 0.0002231034435845,\n",
+       " 'www.theyworkforyou.com': 0.0002208272250724,\n",
+       " 'www.ola.org': 0.000218709862407,\n",
+       " 'www.wikihow.com': 0.0002180965139118,\n",
+       " 'acp.copernicus.org': 0.0002111536489058,\n",
+       " 'www.semanticscholar.org': 0.0001950019041289,\n",
+       " 'www.theatlantic.com': 0.0001913746442804,\n",
+       " 'docplayer.net': 0.0001907412725177,\n",
+       " 'irclogs.ubuntu.com': 0.0001898102449522,\n",
+       " 'www.forbes.com': 0.00018939073865,\n",
+       " 'publications.waset.org': 0.0001873821881616,\n",
+       " 'www.washingtonpost.com': 0.0001866054782221,\n",
+       " 'subsaga.com': 0.0001865089939112,\n",
+       " 'www.nybooks.com': 0.000184557446664,\n",
+       " 'newspapers.library.wales': 0.0001845348751784,\n",
+       " 'www.dailymail.co.uk': 0.0001837949675985,\n",
+       " 'deepai.org': 0.0001831206612305,\n",
+       " 'www.gastearsivi.com': 0.0001817473130172,\n",
+       " 'transparentpng.netlify.app': 0.0001812971084796,\n",
+       " 'api.parliament.uk': 0.0001797907418292,\n",
+       " 'pubmed.ncbi.nlm.nih.gov': 0.0001747089206415,\n",
+       " 'gateway.ipfs.io': 0.0001725078365045,\n",
+       " 'www.thefreelibrary.com': 0.0001708132388158,\n",
+       " 'alumnius.net': 0.0001683239812924,\n",
+       " 'www.yumpu.com': 0.0001673814233117,\n",
+       " 'www.inkitt.com': 0.0001636112276366,\n",
+       " 'm.fanfiction.net': 0.0001552928359265,\n",
+       " 'contracts.justia.com': 0.0001548855632462,\n",
+       " 'patents.google.com': 0.0001517315288932,\n",
+       " 'www.latimes.com': 0.0001510611082861,\n",
+       " 'www.arxiv-vanity.com': 0.0001493404557011,\n",
+       " 'bmcpublichealth.biomedcentral.com': 0.0001492791353507,\n",
+       " 'www.telegraph.co.uk': 0.0001486748451729,\n",
+       " 'archiveofourown.org': 0.0001479185499767,\n",
+       " 'www.economist.com': 0.0001453357732099,\n",
+       " 'studylib.net': 0.0001453062312784,\n",
+       " 'www.espn.com': 0.0001440956536707,\n",
+       " 'hubpages.com': 0.0001390828482346,\n",
+       " 'time.com': 0.000137760710869,\n",
+       " 'www.ipl.org': 0.0001376025183566,\n",
+       " 'robineko.com': 0.0001360150473236,\n",
+       " 'www.advocatekhoj.com': 0.0001359044227581,\n",
+       " 'www.tumgir.com': 0.000135691851343,\n",
+       " 'www.counterpunch.org': 0.0001340337950299,\n",
+       " 'tropedia.fandom.com': 0.000133800765374,\n",
+       " 'www.encyclopedia.com': 0.00013326334508,\n",
+       " 'www.nbcnews.com': 0.0001321288877867,\n",
+       " 'veranstaltungen.badminton.de': 0.0001317963108835,\n",
+       " 'slate.com': 0.0001308241783483,\n",
+       " 'patents.justia.com': 0.0001259783845068,\n",
+       " 'www.british-history.ac.uk': 0.0001259638274029,\n",
+       " 'bmchealthservres.biomedcentral.com': 0.0001243383360853,\n",
+       " 'dokumen.pub': 0.0001212929181714,\n",
+       " 'alchetron.com': 0.0001211202772984,\n",
+       " 'scocal.stanford.edu': 0.000121076402999,\n",
+       " 'www.avclub.com': 0.0001200605774033,\n",
+       " 'www.preceptaustin.org': 0.0001200404127696,\n",
+       " 'www.politico.com': 0.0001198441927229,\n",
+       " 'voracity.e-fic.com': 0.0001192375681888,\n",
+       " 'bmcbioinformatics.biomedcentral.com': 0.0001189960674294,\n",
+       " 'www.smh.com.au': 0.0001184023181027,\n",
+       " 'datatracker.ietf.org': 0.0001181400473718,\n",
+       " 'www.seattletimes.com': 0.0001175724181873,\n",
+       " 'm.scirp.org': 0.0001172250594441,\n",
+       " 'www.independent.co.uk': 0.0001172085630845,\n",
+       " 'www.theglobeandmail.com': 0.0001169944801374,\n",
+       " 'blogs.sap.com': 0.0001168635749453,\n",
+       " 'www.airtasker.com': 0.0001165835971794,\n",
+       " 'bmccancer.biomedcentral.com': 0.0001165253723884,\n",
+       " 'newrepublic.com': 0.0001156806326365,\n",
+       " 'en.wikisource.org': 0.0001146742133368,\n",
+       " 'www.newworldencyclopedia.org': 0.0001145322380761,\n",
+       " 'docs.justia.com': 0.0001144089739136,\n",
+       " 'www.fool.com': 0.0001144039245994,\n",
+       " 'www.vox.com': 0.0001139990805197,\n",
+       " 'moz.com': 0.0001138081019004,\n",
+       " 'www.fastcompany.com': 0.0001133714141558,\n",
+       " 'www.thedailybeast.com': 0.0001121551963952,\n",
+       " 'grantland.com': 0.0001121058124351,\n",
+       " 'apps.apple.com': 0.0001097454882252,\n",
+       " 'www.cdc.gov': 0.0001070063277096,\n",
+       " 'techcrunch.com': 0.0001061237416623,\n",
+       " 'www.bostonglobe.com': 0.0001050931342265,\n",
+       " 'www.creativelive.com': 0.0001047344988024,\n",
+       " 'www.si.com': 0.0001044359259311,\n",
+       " 'devrant.com': 0.0001040909704149,\n",
+       " 'www.opendemocracy.net': 0.0001039552116465,\n",
+       " 'gizmodo.com': 0.0001036007519653,\n",
+       " 'www.healthline.com': 0.0001035295830961,\n",
+       " 'thehockeywriters.com': 0.0001025606947362,\n",
+       " 'www.npr.org': 0.0001020513809621,\n",
+       " 'www.jpost.com': 0.0001018387189275,\n",
+       " 'www.newstatesman.com': 0.0001016472835864,\n",
+       " 'bleacherreport.com': 0.0001013723877576,\n",
+       " 'www.foxnews.com': 0.000101312256334,\n",
+       " 'www.apotelyt.com': 0.0001010690519295,\n",
+       " 'archive.org': 9.968097987253116e-05,\n",
+       " 'www.nzherald.co.nz': 9.956585333416228e-05,\n",
+       " 'truthout.org': 9.943359610027588e-05,\n",
+       " 'www.fictionpress.com': 9.892011745752971e-05,\n",
+       " 'www.politico.eu': 9.790526692680434e-05,\n",
+       " 'www.techradar.com': 9.74725918101852e-05,\n",
+       " 'www.prnewswire.com': 9.738541230153274e-05,\n",
+       " 'www.newyorker.com': 9.735472131572091e-05,\n",
+       " 'www.inquirer.com': 9.707689666725828e-05,\n",
+       " 'www.haaretz.com': 9.671520555620812e-05,\n",
+       " 'digital.bentley.umich.edu': 9.671287844945354e-05,\n",
+       " 'thepointsguy.com': 9.628801110222596e-05,\n",
+       " 'www.thoughtco.com': 9.607799153001185e-05,\n",
+       " 'www.openpr.com': 9.553215812886762e-05,\n",
+       " 'encyclopedia2.thefreedictionary.com': 9.500022574704412e-05,\n",
+       " 'www.usatoday.com': 9.488317082737702e-05,\n",
+       " 'www.democracynow.org': 9.415912889588526e-05,\n",
+       " 'www.cnn.com': 9.4023819599869e-05,\n",
+       " 'www.abc.net.au': 9.389910915143912e-05,\n",
+       " 'www.chicagotribune.com': 9.352145452302566e-05,\n",
+       " 'dictionnaire.sensagent.leparisien.fr': 9.26353234439554e-05,\n",
+       " 'bioresources.cnr.ncsu.edu': 9.263514582988844e-05,\n",
+       " 'contract4j.org': 9.26177287769977e-05,\n",
+       " 'www.motortrend.com': 9.25636108582342e-05,\n",
+       " 'www.canada.ca': 9.242836680820172e-05,\n",
+       " 'www.salon.com': 9.222098969833042e-05,\n",
+       " 'www.thestar.com': 9.19553624240566e-05,\n",
+       " 'www.reuters.com': 9.194116054825516e-05,\n",
+       " 'publications.parliament.uk': 9.19099077220275e-05,\n",
+       " 'tvtropes.org': 9.137703652299302e-05,\n",
+       " 'www.instructables.com': 9.10100169899347e-05,\n",
+       " 'www.stitcher.com': 9.093581418249948e-05,\n",
+       " 'edition.cnn.com': 9.04522725719749e-05,\n",
+       " 'www.inc.com': 9.042346647013876e-05,\n",
+       " 'mickopedia.org': 9.03179963373684e-05,\n",
+       " 'www.rollingstone.com': 8.991108613478614e-05,\n",
+       " 'oregonnews.uoregon.edu': 8.958794815059258e-05,\n",
+       " 'military.wikia.org': 8.950105499931337e-05,\n",
+       " 'supersearch.can.ucsd.edu': 8.913234632023067e-05,\n",
+       " 'www.rt.com': 8.907333857746101e-05,\n",
+       " 'dilbert.com': 8.872705639291213e-05,\n",
+       " 'www.instantcheckmate.com': 8.847558024756348e-05,\n",
+       " 'scienceblogs.com': 8.833675854275406e-05,\n",
+       " 'www.nap.edu': 8.760726132206345e-05,\n",
+       " 'venturebeat.com': 8.734926420311428e-05,\n",
+       " 'www.lastminute.com': 8.725486413892421e-05,\n",
+       " 'www.thesun.co.uk': 8.722505759867021e-05,\n",
+       " 'www.buzzfeednews.com': 8.717305654960247e-05,\n",
+       " 'ilgotha.org': 8.713850880119424e-05,\n",
+       " 'www.finder.com': 8.69887910170946e-05,\n",
+       " 'laagvlieger.x24hr.com': 8.674987109884553e-05,\n",
+       " 'www.cheatsheet.com': 8.580867603424703e-05,\n",
+       " '75-3-247-200.lightspeed.sntcca.sbcglobal.net': 8.560513031353399e-05,\n",
+       " 'themillions.com': 8.537252838138035e-05,\n",
+       " 'www.mic.com': 8.527057065740314e-05,\n",
+       " 'www.baltimoresun.com': 8.519745165825395e-05,\n",
+       " 'www.lrb.co.uk': 8.511623128278618e-05,\n",
+       " 'suptg.thisisnotatrueending.com': 8.504496091984392e-05,\n",
+       " 'www.eurasiareview.com': 8.477297578692904e-05,\n",
+       " 'www.news.com.au': 8.465024084189674e-05,\n",
+       " 'www.firstpost.com': 8.437763224735913e-05,\n",
+       " 'journals.lww.com': 8.429768416857495e-05,\n",
+       " 'insecure.archiveofourown.org': 8.37850319749566e-05,\n",
+       " 'www.bristolpost.co.uk': 8.359137464556217e-05,\n",
+       " 'www.vice.com': 8.343498364723322e-05,\n",
+       " 'kanithamizh.in': 8.289961860170783e-05,\n",
+       " '33.agilestudio.cn': 8.202731242164385e-05,\n",
+       " 'www.colombotelegraph.com': 8.186559662608357e-05,\n",
+       " 'www.icty.org': 8.181046739457158e-05,\n",
+       " 'www.pcmag.com': 8.167827903144583e-05,\n",
+       " 'www.globalsecurity.org': 8.133342138420935e-05,\n",
+       " 'transcripts.cnn.com': 8.113191641287768e-05,\n",
+       " 'www.indiewire.com': 8.110638710933774e-05,\n",
+       " 'www.denofgeek.com': 8.093316264719514e-05,\n",
+       " 'www.smithsonianmag.com': 8.085372203717366e-05,\n",
+       " 'www.bailii.org': 8.072180553231362e-05,\n",
+       " 'www.thisismoney.co.uk': 8.06264956491276e-05,\n",
+       " 'www.bartleby.com': 8.037844491792707e-05,\n",
+       " 'www.express.co.uk': 8.037329773476269e-05,\n",
+       " 'www.csmonitor.com': 8.036398930774429e-05,\n",
+       " 'www.irishexaminer.com': 8.036148096214588e-05,\n",
+       " 'yaledailynews.com': 8.031070146288547e-05,\n",
+       " 'www.texasmonthly.com': 8.021909610166704e-05,\n",
+       " 'fr.slideserve.com': 8.019415401198106e-05,\n",
+       " 'www.speakingtree.in': 8.013157586398822e-05,\n",
+       " 'en.m.wikiquote.org': 7.987411883679364e-05,\n",
+       " 'heavy.com': 7.941452975186778e-05,\n",
+       " 'www.mentalfloss.com': 7.925765303343739e-05,\n",
+       " 'www.romper.com': 7.902143719873806e-05,\n",
+       " 'nypost.com': 7.891734810595757e-05,\n",
+       " 'de.zxc.wiki': 7.869296354298256e-05,\n",
+       " 'www.tripsavvy.com': 7.866977222052784e-05,\n",
+       " 'www.inverse.com': 7.846257634950035e-05,\n",
+       " 'www.ukessays.com': 7.822928208496332e-05,\n",
+       " 'erhverv.studentersamfundet.aau.dk': 7.788662830206765e-05,\n",
+       " 'www.ajmc.com': 7.778079931638652e-05,\n",
+       " 'www.cnet.com': 7.776564049948973e-05,\n",
+       " 'www.aljazeera.com': 7.776164962014891e-05,\n",
+       " 'www.theverge.com': 7.771762670498596e-05,\n",
+       " 'www.kayak.com': 7.758318373064315e-05,\n",
+       " 'homeguides.sfgate.com': 7.750585999031743e-05,\n",
+       " 'medium.com': 7.744013553599594e-05,\n",
+       " 'philarchive.org': 7.742044574800376e-05,\n",
+       " 'wikimili.com': 7.71816454473916e-05,\n",
+       " 'www.hollywoodreporter.com': 7.709207721077766e-05,\n",
+       " 'www.boloji.com': 7.692111098462902e-05,\n",
+       " 'www.readkong.com': 7.682233943952609e-05,\n",
+       " 'parasitesandvectors.biomedcentral.com': 7.654496788817299e-05,\n",
+       " 'shakespir.com': 7.649789653565706e-05,\n",
+       " 'docoh.com': 7.647789501685351e-05,\n",
+       " 'www.thestreet.com': 7.587663152771657e-05,\n",
+       " 'www.quotemaster.org': 7.578018346459137e-05,\n",
+       " 'pizza.dominos.com': 7.554430473414153e-05,\n",
+       " 'www.crikey.com.au': 7.552816722748815e-05,\n",
+       " 'ew.com': 7.54755717150127e-05,\n",
+       " 'aspe.hhs.gov': 7.542470159633038e-05,\n",
+       " 'en.wikibooks.org': 7.538765637665437e-05,\n",
+       " 'www.deseret.com': 7.517427301149898e-05,\n",
+       " 'www.huffingtonpost.co.uk': 7.506976706937772e-05,\n",
+       " 'thehill.com': 7.506696149207544e-05,\n",
+       " 'www.businesswire.com': 7.447800774521049e-05,\n",
+       " 'finance.yahoo.com': 7.432127601785517e-05,\n",
+       " 'www.patheos.com': 7.431243156227694e-05,\n",
+       " 'gahistoricnewspapers.galileo.usg.edu': 7.409300569406873e-05,\n",
+       " 'www.thisdaylive.com': 7.401369920079166e-05,\n",
+       " 'www.hindustantimes.com': 7.400497073807348e-05,\n",
+       " 'americanliterature.com': 7.39991203481951e-05,\n",
+       " 'www.govinfo.gov': 7.398683597936124e-05,\n",
+       " 'www.vulture.com': 7.397641837061869e-05,\n",
+       " 'www.verywellhealth.com': 7.367888943505635e-05,\n",
+       " 'edwardbetts.com': 7.347130933767997e-05,\n",
+       " 'www.osapublishing.org': 7.340340639245581e-05,\n",
+       " 'jamanetwork.com': 7.339769374409872e-05,\n",
+       " 'worldcrunch.com': 7.332037362854988e-05,\n",
+       " 'journals.biologists.com': 7.314969738455134e-05,\n",
+       " 'www.knowpia.com': 7.278462073189873e-05,\n",
+       " 'www.japantimes.co.jp': 7.270068177377371e-05,\n",
+       " 'casestudylion.com': 7.267251725744468e-05,\n",
+       " 'listverse.com': 7.263129629480715e-05,\n",
+       " 'www.wired.com': 7.262107079923904e-05,\n",
+       " 'reason.com': 7.227048600453796e-05,\n",
+       " 'www.pcworld.com': 7.210347803473826e-05,\n",
+       " 'lrb.co.uk': 7.202621229084256e-05,\n",
+       " 'apnews.com': 7.177186894698473e-05,\n",
+       " 'vault.si.com': 7.175390092800879e-05,\n",
+       " 'www.mirror.co.uk': 7.167631257897111e-05,\n",
+       " 'www.washingtontimes.com': 7.157697919345235e-05,\n",
+       " 'abcnews.go.com': 7.154017320905017e-05,\n",
+       " 'www.michigandaily.com': 7.145109069253766e-05,\n",
+       " 'japantoday.com': 7.141615509300366e-05,\n",
+       " 'www.indystar.com': 7.137189294256688e-05,\n",
+       " 'www.alternet.org': 7.133872260937165e-05,\n",
+       " 'brooklynrail.org': 7.131313530940169e-05,\n",
+       " 'www.angelfire.com': 7.12078065429295e-05,\n",
+       " 'www.cracked.com': 7.115592873627557e-05,\n",
+       " 'www.nj.com': 7.110599381002733e-05,\n",
+       " 'washingtoncitypaper.com': 7.102516853523909e-05,\n",
+       " 'www.lifehack.org': 7.096077437403122e-05,\n",
+       " 'www.cbc.ca': 7.093102945498411e-05,\n",
+       " 'www.etsy.com': 7.086754148799531e-05,\n",
+       " 'www.wnd.com': 7.084338235011462e-05,\n",
+       " 'thenextweb.com': 7.078859022285212e-05,\n",
+       " 'www.paperdue.com': 7.078620874444437e-05,\n",
+       " 'www.intechopen.com': 7.062377161828572e-05,\n",
+       " 'core-cms.prod.aop.cambridge.org': 7.005774096038663e-05,\n",
+       " 'www.foxbusiness.com': 6.999398113513212e-05,\n",
+       " 'www.freep.com': 6.994770723352898e-05,\n",
+       " 'www.rxlist.com': 6.978336709951025e-05,\n",
+       " 'variety.com': 6.975902309800886e-05,\n",
+       " 'news.yahoo.com': 6.969056918669951e-05,\n",
+       " 'thoughtcatalog.com': 6.942696816269956e-05,\n",
+       " 'www.afr.com': 6.928289415619634e-05,\n",
+       " 'www.frontstretch.com': 6.924485212287934e-05,\n",
+       " 'www.yahoo.com': 6.923795054770682e-05,\n",
+       " 'counter-currents.com': 6.914461254314151e-05,\n",
+       " 'www.macworld.com': 6.895888622555247e-05,\n",
+       " 'www.actionnetwork.com': 6.890092604330007e-05,\n",
+       " 'www.thenation.com': 6.889150162342162e-05,\n",
+       " 'supreme.justia.com': 6.858906111519188e-05,\n",
+       " 'www.technologyreview.com': 6.842741419039226e-05,\n",
+       " 'www.bbc.com': 6.82069081386749e-05,\n",
+       " 'www.answers.com': 6.810150325188831e-05,\n",
+       " 'www.tomsguide.com': 6.802705033484864e-05,\n",
+       " 'qz.com': 6.796972086377271e-05,\n",
+       " 'locations.acima.com': 6.781505888401377e-05,\n",
+       " 'www.pajiba.com': 6.774461134542243e-05,\n",
+       " 'www.slideserve.com': 6.767139447729758e-05,\n",
+       " 'www.cinemablend.com': 6.707308880609173e-05,\n",
+       " 'www.scoop.co.nz': 6.700737160132399e-05,\n",
+       " 'www.desmoinesregister.com': 6.696830375615098e-05,\n",
+       " 'nationalpost.com': 6.695414175289518e-05,\n",
+       " 'www.edweek.org': 6.689663466775233e-05,\n",
+       " 'www.yukon-news.com': 6.681017648969827e-05,\n",
+       " 'bmcinfectdis.biomedcentral.com': 6.676929263131023e-05,\n",
+       " 'hbr.org': 6.664135975623728e-05,\n",
+       " 'www.zdnet.com': 6.661904562978665e-05,\n",
+       " 'www.lifesitenews.com': 6.656624712980612e-05,\n",
+       " 'www.lifenews.com': 6.648742635663048e-05,\n",
+       " 'projects.sare.org': 6.648619393249253e-05,\n",
+       " 'www.aanda.org': 6.623126337478073e-05,\n",
+       " 'www.thenationalnews.com': 6.621839179209292e-05,\n",
+       " 'www.digitaltrends.com': 6.600602698923979e-05,\n",
+       " 'dzone.com': 6.590109332344712e-05,\n",
+       " 'cc.bingj.com': 6.589474996391355e-05,\n",
+       " 'www.cnbc.com': 6.588816736910614e-05,\n",
+       " 'www.today.com': 6.586893792777725e-05,\n",
+       " 'www.tor.com': 6.58186260247339e-05,\n",
+       " 'www.scotsman.com': 6.567917723352487e-05,\n",
+       " 'incidecoder.com': 6.564429238086714e-05,\n",
+       " 'nymag.com': 6.561455833615067e-05,\n",
+       " 'www.pbs.org': 6.54957526492523e-05,\n",
+       " 'www.entrepreneur.com': 6.546032408006312e-05,\n",
+       " 'www.aol.com': 6.534515766914859e-05,\n",
+       " 'deadline.com': 6.521372688439e-05,\n",
+       " 'www.azcentral.com': 6.515080438259391e-05,\n",
+       " 'chicagoreader.com': 6.509821611967221e-05,\n",
+       " 'www.plymouthherald.co.uk': 6.485229313249809e-05,\n",
+       " 'www.inman.com': 6.484517044593611e-05,\n",
+       " 'www.rediff.com': 6.465703727650121e-05,\n",
+       " 'github.com': 6.455723629476541e-05,\n",
+       " 'www.theepochtimes.com': 6.435335346980286e-05,\n",
+       " 'indianexpress.com': 6.421343345759994e-05,\n",
+       " 'smallbusiness.chron.com': 6.420471949398925e-05,\n",
+       " 'todayinsci.com': 6.418823400875574e-05,\n",
+       " 'www.biblegateway.com': 6.416743866381627e-05,\n",
+       " 'www.computerweekly.com': 6.40963241662797e-05,\n",
+       " 'electricscotland.com': 6.40196528857917e-05,\n",
+       " 'www.eurekalert.org': 6.398032768146048e-05,\n",
+       " 'chestofbooks.com': 6.388934940664165e-05,\n",
+       " 'wwd.com': 6.380502622216775e-05,\n",
+       " 'www.vanityfair.com': 6.37191190101989e-05,\n",
+       " 'www.detroitnews.com': 6.369564133037095e-05,\n",
+       " 'www.popsci.com': 6.354896110929367e-05,\n",
+       " 'email.varsitytutors.com': 6.349104442436378e-05,\n",
+       " 'unispal.un.org': 6.334205522041628e-05,\n",
+       " 'www.stylist.co.uk': 6.33394961279416e-05,\n",
+       " 'www.polygon.com': 6.3172749142077e-05,\n",
+       " 'spectator.org': 6.305600595322126e-05,\n",
+       " 'www.trustedreviews.com': 6.28778300458656e-05,\n",
+       " 'matadornetwork.com': 6.286947493516568e-05,\n",
+       " 'www.bizpacreview.com': 6.274636663811513e-05,\n",
+       " 'cloudflare-ipfs.com': 6.264667439968561e-05,\n",
+       " 'moderndiplomacy.eu': 6.260917608290006e-05,\n",
+       " 'www.cyclingnews.com': 6.254338275781793e-05,\n",
+       " 'www.orlandosentinel.com': 6.235571718460006e-05,\n",
+       " 'torontosun.com': 6.229324415513663e-05,\n",
+       " 'hindi.hwnews.in': 6.223899936918246e-05,\n",
+       " 'www.nakedcapitalism.com': 6.218519318123032e-05,\n",
+       " 'www.golfdigest.com': 6.213693290189896e-05,\n",
+       " 'www.indiatoday.in': 6.208597578857161e-05,\n",
+       " 'admin.rushlimbaugh.com': 6.205893857785112e-05,\n",
+       " 'slideplayer.com': 6.194997416017198e-05,\n",
+       " 'koreajoongangdaily.joins.com': 6.189866544348762e-05,\n",
+       " 'www.discovermagazine.com': 6.185195294388246e-05,\n",
+       " 'www.ncronline.org': 6.178336491582866e-05,\n",
+       " 'scroll.in': 6.174224182216679e-05,\n",
+       " 'www.elephantjournal.com': 6.153964579299534e-05,\n",
+       " 'www.eadt.co.uk': 6.147534950076313e-05,\n",
+       " 'www.mlb.com': 6.127388802675397e-05,\n",
+       " 'signalscv.com': 6.126671096853886e-05,\n",
+       " 'www.genomeweb.com': 6.11374333012448e-05,\n",
+       " 'sabr.org': 6.11251996792872e-05,\n",
+       " 'www.liverpoolecho.co.uk': 6.105908374906308e-05,\n",
+       " 'www.tomshardware.com': 6.105648115926588e-05,\n",
+       " 'blogs.lse.ac.uk': 6.087712357464855e-05,\n",
+       " 'lawprofessors.typepad.com': 6.073721806155311e-05,\n",
+       " 'www.walesonline.co.uk': 6.073342654494048e-05,\n",
+       " 'www.rferl.org': 6.072680770236432e-05,\n",
+       " 'www.treatwell.co.uk': 6.070949214322612e-05,\n",
+       " 'www.straitstimes.com': 6.070556650986906e-05,\n",
+       " 'wccftech.com': 6.061624113330584e-05,\n",
+       " 'genomebiology.biomedcentral.com': 6.0551513492625336e-05,\n",
+       " 'www.refinery29.com': 6.031577612847368e-05,\n",
+       " 'profiles.stanford.edu': 6.031038246048171e-05,\n",
+       " 'www.rollcall.com': 6.0308276465116567e-05,\n",
+       " 'wol.jw.org': 6.020821812422255e-05,\n",
+       " 'translational-medicine.biomedcentral.com': 6.0120034552375366e-05,\n",
+       " 'www.sun-sentinel.com': 6.006348803310472e-05,\n",
+       " 'www.varsitytutors.com': 6.002798696837802e-05,\n",
+       " 'salesandmarketingnetwork.com': 5.997694286040564e-05,\n",
+       " 'www.nottinghampost.com': 5.986280951090086e-05,\n",
+       " 'www.dailypress.com': 5.9842826115981694e-05,\n",
+       " 'thestuffedbakedpotatofactory.com': 5.982481459968324e-05,\n",
+       " 'www.catholicculture.org': 5.978192986445947e-05,\n",
+       " 'www.wsws.org': 5.972766695462091e-05,\n",
+       " 'www.irishcentral.com': 5.962983060195208e-05,\n",
+       " 'db0nus869y26v.cloudfront.net': 5.961873153515678e-05,\n",
+       " 'www.brookings.edu': 5.9409991509179976e-05,\n",
+       " 'www.tripadvisor.com': 5.919981969633702e-05,\n",
+       " 'docs.microsoft.com': 5.914151153550449e-05,\n",
+       " 'www.courant.com': 5.906802280911392e-05,\n",
+       " 'www.gearpatrol.com': 5.8947230744487336e-05,\n",
+       " 'www.thefreedictionary.com': 5.8906499126728096e-05,\n",
+       " 'www.dailyrecord.co.uk': 5.889077121986173e-05,\n",
+       " 'www.hulldailymail.co.uk': 5.880339234848109e-05,\n",
+       " 'www.mycentraljersey.com': 5.868136785971656e-05,\n",
+       " 'www.stokesentinel.co.uk': 5.8600274351439456e-05,\n",
+       " 'bookriot.com': 5.853559020808147e-05,\n",
+       " 'wn.rsarchive.org': 5.845390223639666e-05,\n",
+       " 'chicago.suntimes.com': 5.844645694469269e-05,\n",
+       " 'info.mzalendo.com': 5.843388621848561e-05,\n",
+       " 'www.manchestereveningnews.co.uk': 5.820470970048009e-05,\n",
+       " 'malariajournal.biomedcentral.com': 5.816658792207181e-05,\n",
+       " 'quizlet.com': 5.811156018431234e-05,\n",
+       " 'www.cyclingweekly.com': 5.804181947721189e-05,\n",
+       " 'www.tw.kayak.com': 5.791512627583065e-05,\n",
+       " 'www.opentable.com': 5.791108464961355e-05,\n",
+       " 'www.charityjob.co.uk': 5.780437121837461e-05,\n",
+       " 'thecyberwire.com': 5.769611000740947e-05,\n",
+       " 'bmcresnotes.biomedcentral.com': 5.769451873036076e-05,\n",
+       " 'dev.to': 5.768846172820043e-05,\n",
+       " 'newsblaze.com': 5.767688781563433e-05,\n",
+       " 'mmajunkie.usatoday.com': 5.767364001555314e-05,\n",
+       " 'villains.fandom.com': 5.756588627335069e-05,\n",
+       " 'ar15partspro.com': 5.756451973246832e-05,\n",
+       " 'www.informationweek.com': 5.746075324482982e-05,\n",
+       " 'www.phillymag.com': 5.737931538274948e-05,\n",
+       " 'www.kayak.co.uk': 5.732462837401639e-05,\n",
+       " 'www.mdpi.com': 5.7296170250760384e-05,\n",
+       " 'www.villagevoice.com': 5.728727504830589e-05,\n",
+       " 'www.smore.com': 5.721818680104317e-05,\n",
+       " 'www.outsideonline.com': 5.721286562858873e-05,\n",
+       " 'bgr.com': 5.711787472576781e-05,\n",
+       " 'www.livemint.com': 5.711578685428705e-05,\n",
+       " 'dir.indiamart.com': 5.711059254902328e-05,\n",
+       " 'timelines.ws': 5.705113170914408e-05,\n",
+       " 'www.medicalnewstoday.com': 5.698305477462985e-05,\n",
+       " 'www.gty.org': 5.697810695419367e-05,\n",
+       " 'anchor.fm': 5.696123724261127e-05,\n",
+       " 'investorplace.com': 5.6919421816566006e-05,\n",
+       " 'www.medpagetoday.com': 5.681609030215267e-05,\n",
+       " 'www.barnesandnoble.com': 5.678419951519478e-05,\n",
+       " 'www.foxsports.com': 5.676209562580296e-05,\n",
+       " 'www.wrswebsolutions.ca': 5.674841571787172e-05,\n",
+       " 'www.saltwire.com': 5.671346199445334e-05,\n",
+       " 'www.livescience.com': 5.670516125540656e-05,\n",
+       " 'www.myantispyware.com': 5.6697172247171146e-05,\n",
+       " 'www.sunnewsonline.com': 5.663415912595314e-05,\n",
+       " 'yourstory.com': 5.658385447246354e-05,\n",
+       " 'www.edp24.co.uk': 5.654794743272668e-05,\n",
+       " 'sports.yahoo.com': 5.648601811979471e-05,\n",
+       " 'www.bostonmagazine.com': 5.643611944131524e-05,\n",
+       " 'everything.explained.today': 5.626292397738766e-05,\n",
+       " 'wattsupwiththat.com': 5.6136426513957744e-05,\n",
+       " 'www.findlaw.com': 5.608533165910909e-05,\n",
+       " 'www.indiatimes.com': 5.608401586510299e-05,\n",
+       " 'www.businessinsider.com.au': 5.606848007141107e-05,\n",
+       " 'www.booking.com': 5.598988403440176e-05,\n",
+       " 'metro.co.uk': 5.5747183473870686e-05,\n",
+       " 'www.chanrobles.com': 5.571989252876885e-05,\n",
+       " 'jezebel.com': 5.567175549185128e-05,\n",
+       " 'fliphtml5.com': 5.56577602283318e-05,\n",
+       " 'familypedia.wikia.org': 5.564669015975151e-05,\n",
+       " 'thefederalist.com': 5.5615009609852446e-05,\n",
+       " 'www.breitbart.com': 5.558815726275264e-05,\n",
+       " 'www.law.cornell.edu': 5.553168686379641e-05,\n",
+       " 'arthritis-research.biomedcentral.com': 5.548828378547932e-05,\n",
+       " 'www.northjersey.com': 5.548443427243666e-05,\n",
+       " 'www.mailtribune.com': 5.546396515741607e-05,\n",
+       " 'freerepublic.com': 5.539630144746575e-05,\n",
+       " 'philpapers.org': 5.535732784649152e-05,\n",
+       " 'www.investopedia.com': 5.5335742300193024e-05,\n",
+       " 'groups.google.com': 5.529671795234253e-05,\n",
+       " 'www.poughkeepsiejournal.com': 5.527307353277827e-05,\n",
+       " 'www.internetnews.com': 5.525952411681458e-05,\n",
+       " 'dailyuknews.com': 5.517654209978492e-05,\n",
+       " 'www.aniapalka.pl': 5.5066240139438416e-05,\n",
+       " 'www.expedia.com': 5.505563766707517e-05,\n",
+       " 'www.greenbaypressgazette.com': 5.502396799150673e-05,\n",
+       " 'inews.co.uk': 5.4974982756800105e-05,\n",
+       " 'www.birminghammail.co.uk': 5.4962364908493626e-05,\n",
+       " 'doczz.net': 5.495750770747935e-05,\n",
+       " 'www.bookstrand.com': 5.491478971199189e-05,\n",
+       " 'redstate.com': 5.4876149590490285e-05,\n",
+       " 'literatureessaysamples.com': 5.486643881323862e-05,\n",
+       " 'www.runnersworld.com': 5.48382416739177e-05,\n",
+       " 'whyy.org': 5.4829654577497694e-05,\n",
+       " 'www.cfr.org': 5.481779793233525e-05,\n",
+       " 'www.tcrecord.org': 5.470973245932143e-05,\n",
+       " 'www.modernghana.com': 5.461470893350862e-05,\n",
+       " 'vdare.com': 5.460818433513124e-05,\n",
+       " 'allthetropes.org': 5.445298226361772e-05,\n",
+       " 'www.democratandchronicle.com': 5.441863387793769e-05,\n",
+       " 'www.assignmentpoint.com': 5.432789846316957e-05,\n",
+       " 'townhall.com': 5.429902349057279e-05,\n",
+       " 'thediplomat.com': 5.428771781149553e-05,\n",
+       " 'www.newspapers.com': 5.424167589561248e-05,\n",
+       " 'www.lifewire.com': 5.418477777298484e-05,\n",
+       " 'apk-dl.com': 5.4162264283806e-05,\n",
+       " 'twit.tv': 5.412131517943417e-05,\n",
+       " 'ng.opera.news': 5.40462967972018e-05,\n",
+       " 'www.theage.com.au': 5.401626552078146e-05,\n",
+       " 'www.sfexaminer.com': 5.390221916592171e-05,\n",
+       " 'www.informit.com': 5.381701516066687e-05,\n",
+       " 'munafa.co.com': 5.380871079684321e-05,\n",
+       " 'paperzz.com': 5.357548902784372e-05,\n",
+       " 'www.psychologytoday.com': 5.35142085499726e-05,\n",
+       " 'www.sandiegouniontribune.com': 5.347108820425187e-05,\n",
+       " 'www.mybookie.ag': 5.343941127912968e-05,\n",
+       " 'www.bbc.co.uk': 5.3411145269048114e-05,\n",
+       " 'www.leicestermercury.co.uk': 5.33733497205587e-05,\n",
+       " 'www.simonandschuster.com': 5.336544770696831e-05,\n",
+       " 'laitman.com': 5.328105202695689e-05,\n",
+       " 'www.startribune.com': 5.323949758484672e-05,\n",
+       " 'www.thedrive.com': 5.314180622325294e-05,\n",
+       " 'www.tennessean.com': 5.312762609611277e-05,\n",
+       " 'www.premiumtimesng.com': 5.306046985492513e-05,\n",
+       " 'www.gsmarena.com': 5.2984045059264736e-05,\n",
+       " 'observer.com': 5.296911822808804e-05,\n",
+       " 'www.jsonline.com': 5.2942515990592704e-05,\n",
+       " 'mronline.org': 5.2934798840623017e-05,\n",
+       " 'dailyreckoning.com': 5.292931092843226e-05,\n",
+       " 'theconversation.com': 5.291890056924346e-05,\n",
+       " 'arcus-www.amazon.com': 5.29151742986146e-05,\n",
+       " 'www.scientificamerican.com': 5.284368282428288e-05,\n",
+       " 'www.cornwalllive.com': 5.2795886516391694e-05,\n",
+       " 'www.cambridge-news.co.uk': 5.2745951590143466e-05,\n",
+       " 'www.wetwebmedia.com': 5.268803490521358e-05,\n",
+       " 'pitchfork.com': 5.266239323359047e-05,\n",
+       " 'www.apartmentlist.com': 5.241080834493553e-05,\n",
+       " 'www.washingtonian.com': 5.231334896906184e-05,\n",
+       " 'forward.com': 5.2309702443524256e-05,\n",
+       " 'content.iospress.com': 5.225874895497378e-05,\n",
+       " 'www.thecanary.co': 5.2257574527265855e-05,\n",
+       " 'www.delawareonline.com': 5.202899247266805e-05,\n",
+       " 'www.crosswalk.com': 5.192594369085107e-05,\n",
+       " 'warwick.ac.uk': 5.185486181630638e-05,\n",
+       " 'newatlas.com': 5.182483053988604e-05,\n",
+       " 'www.europeanfinancialreview.com': 5.182141962484542e-05,\n",
+       " 'conwebwatch.tripod.com': 5.178665438982461e-05,\n",
+       " 'www.devonlive.com': 5.168742612283527e-05,\n",
+       " 'www.valuewalk.com': 5.1608906206140365e-05,\n",
+       " 'www.softpanorama.org': 5.160061271664734e-05,\n",
+       " 'www.capitalgazette.com': 5.1599318671302486e-05,\n",
+       " 'www.esquire.com': 5.158403661199192e-05,\n",
+       " 'www.essexlive.news': 5.146875058344046e-05,\n",
+       " 'www.space.com': 5.145492205965729e-05,\n",
+       " 'pjmedia.com': 5.136386766452406e-05,\n",
+       " 'wwwnc-origin.cdc.gov': 5.127543760784928e-05,\n",
+       " 'www.dovepress.com': 5.126935160747394e-05,\n",
+       " 'www.al-islam.org': 5.120930717851765e-05,\n",
+       " 'www.derbytelegraph.co.uk': 5.118368000600204e-05,\n",
+       " 'www.motorbiscuit.com': 5.115163697841535e-05,\n",
+       " 'www.mercurynews.com': 5.1100908226031215e-05,\n",
+       " 'www.texasobserver.org': 5.097951807322003e-05,\n",
+       " 'hess.copernicus.org': 5.095490946300668e-05,\n",
+       " 'www.distractify.com': 5.082006413843059e-05,\n",
+       " 'www.2-spyware.com': 5.0796260228683775e-05,\n",
+       " 'reliefweb.int': 5.074167833848011e-05,\n",
+       " 'www.dailypost.co.uk': 5.074157321995069e-05,\n",
+       " 'dailycaller.com': 5.073757509105611e-05,\n",
+       " 'www.ipswichstar.co.uk': 5.0712806990660194e-05,\n",
+       " 'www.somersetlive.co.uk': 5.070674998849986e-05,\n",
+       " 'www.newsreview.com': 5.05868423694311e-05,\n",
+       " '0-bmcpublichealth-biomedcentral-com.brum.beds.ac.uk': 5.052361176160052e-05,\n",
+       " 'www.chroniclelive.co.uk': 5.049609245555548e-05,\n",
+       " 'www.bikeradar.com': 5.030546906440653e-05,\n",
+       " 'www.gamesradar.com': 5.025984762264113e-05,\n",
+       " 'www.scielo.br': 5.0237004278766544e-05,\n",
+       " 'en.m.wikisource.org': 5.021848166892853e-05,\n",
+       " 'mail.python.org': 5.018190404546956e-05,\n",
+       " 'www.standardmedia.co.ke': 5.01001617021316e-05,\n",
+       " 'bankrupt.com': 5.0096000458277584e-05,\n",
+       " 'nationalinterest.org': 5.007328760637054e-05,\n",
+       " 'betches.com': 4.9974026716389314e-05,\n",
+       " 'ke.opera.news': 4.984810921725961e-05,\n",
+       " 'hpathy.com': 4.976094420771467e-05,\n",
+       " 'www.knoxnews.com': 4.975990389675117e-05,\n",
+       " 'economictimes.indiatimes.com': 4.974863809021956e-05,\n",
+       " 'steelersdepot.com': 4.974135228869815e-05,\n",
+       " 'www.thequint.com': 4.970924763990456e-05,\n",
+       " 'traveltriangle.com': 4.9628802966688325e-05,\n",
+       " 'www.publishersweekly.com': 4.960210286021733e-05,\n",
+       " 'www.monstersandcritics.com': 4.9594179097965685e-05,\n",
+       " 'elispot.biz': 4.957056730139332e-05,\n",
+       " 'www.educba.com': 4.955667715640325e-05,\n",
+       " 'www.embedded.com': 4.954561071259984e-05,\n",
+       " 'hyperallergic.com': 4.949491095843071e-05,\n",
+       " 'www.pdf-archive.com': 4.949371478206152e-05,\n",
+       " 'nautilus.org': 4.947787813188887e-05,\n",
+       " 'bmccomplementmedtherapies.biomedcentral.com': 4.944925689367342e-05,\n",
+       " 'traveltips.usatoday.com': 4.9349629901227686e-05,\n",
+       " 'bg.copernicus.org': 4.9268152166601696e-05,\n",
+       " 'www.theregister.com': 4.925028201660143e-05,\n",
+       " 'www.rockpapershotgun.com': 4.924633825935999e-05,\n",
+       " 'www.autostraddle.com': 4.918608721812175e-05,\n",
+       " 'www.mylondon.news': 4.917486853368953e-05,\n",
+       " 'www.thecut.com': 4.917366148298971e-05,\n",
+       " 'www.sellswatches.com': 4.907603174260283e-05,\n",
+       " 'www.siliconrepublic.com': 4.906262731771419e-05,\n",
+       " 'www.pearltrees.com': 4.904416632908308e-05,\n",
+       " 'www.artofmanliness.com': 4.898196515788537e-05,\n",
+       " 'top.hatnote.com': 4.89120142137261e-05,\n",
+       " 'www.pastemagazine.com': 4.886283686584502e-05,\n",
+       " 'www.hamhigh.co.uk': 4.883906920386697e-05,\n",
+       " 'movieweb.com': 4.875867890230389e-05,\n",
+       " 'www.opednews.com': 4.8749062369251e-05,\n",
+       " 'www.thetruthaboutguns.com': 4.873130096255702e-05,\n",
+       " 'www.newsday.com': 4.872338082508225e-05,\n",
+       " 'theinfolist.com': 4.87136519239462e-05,\n",
+       " 'www.menshealth.com': 4.870897958655262e-05,\n",
+       " 'www.lancs.live': 4.8657065532129926e-05,\n",
+       " 'tribune.com.pk': 4.86490004035801e-05,\n",
+       " 'www.soapcentral.com': 4.85898766679504e-05,\n",
+       " 'www.aafp.org': 4.856965041298052e-05,\n",
+       " 'www.thetoptens.com': 4.855074357679362e-05,\n",
+       " 'www.yorkdispatch.com': 4.85267113061036e-05,\n",
+       " 'www.gloucestershirelive.co.uk': 4.8502167941874014e-05,\n",
+       " 'www.the-sun.com': 4.844362417054453e-05,\n",
+       " 'www.routledge.com': 4.8429161310808e-05,\n",
+       " 'english.pravda.ru': 4.841194362064546e-05,\n",
+       " 'online.flippingbook.com': 4.835702825096917e-05,\n",
+       " 'www.hurriyetdailynews.com': 4.828615661348331e-05,\n",
+       " 'newyork.cbslocal.com': 4.8281741635247945e-05,\n",
+       " 'www.gov.scot': 4.827646033533914e-05,\n",
+       " 'bmcmicrobiol.biomedcentral.com': 4.8238701034618486e-05,\n",
+       " 'starwars.fandom.com': 4.819857837937447e-05,\n",
+       " 'www.nasdaq.com': 4.818602215227488e-05,\n",
+       " 'www.buzzsprout.com': 4.817318319257895e-05,\n",
+       " 'www.sensesofcinema.com': 4.812550650232467e-05,\n",
+       " 'infogalactic.com': 4.811656417777078e-05,\n",
+       " 'www.desertsun.com': 4.807424853251659e-05,\n",
+       " 'www.rushlimbaugh.com': 4.806293197910871e-05,\n",
+       " 'www.app.com': 4.7993905353052895e-05,\n",
+       " 'www.tallahassee.com': 4.795652302912738e-05,\n",
+       " 'enhancedodds.co.uk': 4.793052612937039e-05,\n",
+       " 'legislature.idaho.gov': 4.789320905142864e-05,\n",
+       " 'www.examinerlive.co.uk': 4.7862438320525536e-05,\n",
+       " 'dailyfeed.dailynewsegypt.com': 4.786186560577908e-05,\n",
+       " 'www.wfae.org': 4.77679258882523e-05,\n",
+       " 'timesofindia.indiatimes.com': 4.771500414585798e-05,\n",
+       " 'www.msnbc.com': 4.7688260542064473e-05,\n",
+       " 'guardian.ng': 4.76175882673068e-05,\n",
+       " 'www.thewrap.com': 4.7593280513574185e-05,\n",
+       " 'www.hercampus.com': 4.75635319697502e-05,\n",
+       " 'uproxx.com': 4.745981260421109e-05,\n",
+       " 'www.sunsigns.org': 4.745620957599603e-05,\n",
+       " 'www.enr.com': 4.741904473868309e-05,\n",
+       " 'www.pcgamer.com': 4.731802583191685e-05,\n",
+       " 'www.drugs.com': 4.728373181788996e-05,\n",
+       " 'www.outlookindia.com': 4.726324457898498e-05,\n",
+       " '0-bmcplantbiol-biomedcentral-com.brum.beds.ac.uk': 4.725807564715934e-05,\n",
+       " 'www2.deloitte.com': 4.723304293805146e-05,\n",
+       " 'bible.knowing-jesus.com': 4.7134862231579375e-05,\n",
+       " 'www.rawstory.com': 4.704613131842386e-05,\n",
+       " 'www.niche.com': 4.702445515270346e-05,\n",
+       " 'www.courier-journal.com': 4.690736036049072e-05,\n",
+       " 'www.phoenixnewtimes.com': 4.6901285234446e-05,\n",
+       " 'news.thomasnet.com': 4.689956709020663e-05,\n",
+       " 'www.pulselive.co.ke': 4.689523548183942e-05,\n",
+       " 'legalinsurrection.com': 4.687433501837054e-05,\n",
+       " 'www.enotes.com': 4.6863946407843e-05,\n",
+       " 'www.lansingstatejournal.com': 4.684520631139241e-05,\n",
+       " 'www.prweb.com': 4.680034244799416e-05,\n",
+       " 'www.kitces.com': 4.665628656537534e-05,\n",
+       " 'www.churchofjesuschrist.org': 4.647538482580869e-05,\n",
+       " 'www.gazettelive.co.uk': 4.647152081365853e-05,\n",
+       " 'webstatsdomain.org': 4.644640473468249e-05,\n",
+       " 'bmcpregnancychildbirth.biomedcentral.com': 4.641730865069624e-05,\n",
+       " 'docs.google.com': 4.640923989736955e-05,\n",
+       " 'www.popularmechanics.com': 4.636077300575624e-05,\n",
+       " 'www.flavorwire.com': 4.633587803816965e-05,\n",
+       " 'dailymed.nlm.nih.gov': 4.632968329448802e-05,\n",
+       " 'www.bmj.com': 4.631275558647559e-05,\n",
+       " 'ca10.washburnlaw.edu': 4.63113600473782e-05,\n",
+       " 'interreviewed.com': 4.623577619995312e-05,\n",
+       " 'www.t3.com': 4.622088924132206e-05,\n",
+       " 'www.yorkshirepost.co.uk': 4.618057809768047e-05,\n",
+       " 'bestfamilypets.com': 4.617664521476966e-05,\n",
+       " 'www.dawn.com': 4.617599275493192e-05,\n",
+       " 'www.talkers.com': 4.614886130001265e-05,\n",
+       " 'lawandcrime.com': 4.613004870802453e-05,\n",
+       " 'www.mlive.com': 4.607975492886555e-05,\n",
+       " 'moam.info': 4.605280471279009e-05,\n",
+       " 'www.beliefnet.com': 4.604886095554865e-05,\n",
+       " 'www.yerepouni-news.com': 4.602874706866193e-05,\n",
+       " 'www.tbsnews.net': 4.599745074511176e-05,\n",
+       " 'www.getsurrey.co.uk': 4.59839267025862e-05,\n",
+       " 'www.rte.ie': 4.5900487963670126e-05,\n",
+       " 'soccer.nbcsports.com': 4.5871112771864406e-05,\n",
+       " 'www.usforacle.com': 4.586845218563718e-05,\n",
+       " 'www.gizmodo.com.au': 4.586498689894342e-05,\n",
+       " 'www.phnompenhpost.com': 4.57980118965996e-05,\n",
+       " 'www.bodybuilding.com': 4.57621882267309e-05,\n",
+       " 'lithub.com': 4.574725052122357e-05,\n",
+       " 'www.pharmtech.com': 4.567671236321033e-05,\n",
+       " 'www.rlpmax.ca': 4.5658856712317565e-05,\n",
+       " 'www.dezeen.com': 4.56107957957144e-05,\n",
+       " 'foreignpolicy.com': 4.560099439904082e-05,\n",
+       " 'sfbayview.com': 4.558094938291475e-05,\n",
+       " 'www.westword.com': 4.5521615410226224e-05,\n",
+       " 'iasbaba.com': 4.552126380686922e-05,\n",
+       " 'tgdaily.com': 4.551642472973933e-05,\n",
+       " 'www.history.navy.mil': 4.549480293567206e-05,\n",
+       " 'www.vcstar.com': 4.548789411094579e-05,\n",
+       " 'za.opera.news': 4.538702744480835e-05,\n",
+       " 'www.thescottishsun.co.uk': 4.538542529342902e-05,\n",
+       " 'www.huntspost.co.uk': 4.5374753950305125e-05,\n",
+       " 'www.houseofnames.com': 4.531766371450304e-05,\n",
+       " 'www.legal500.com': 4.528724458695694e-05,\n",
+       " 'www.iranicaonline.org': 4.524300056040454e-05,\n",
+       " 'www.lexico.com': 4.523562051468435e-05,\n",
+       " 'biotechnologyforbiofuels.biomedcentral.com': 4.5124981450088345e-05,\n",
+       " 'www.wjgnet.com': 4.511053308945932e-05,\n",
+       " 'www.thenewsstar.com': 4.498095094090765e-05,\n",
+       " 'www.mediapost.com': 4.4971530145806087e-05,\n",
+       " 'www.nhpr.org': 4.492473790110963e-05,\n",
+       " 'historicengland.org.uk': 4.490641102922295e-05,\n",
+       " 'ausbcomp.com': 4.484445634285284e-05,\n",
+       " 'byjus.com': 4.483729378374522e-05,\n",
+       " 'www.electricscotland.com': 4.483225171911026e-05,\n",
+       " 'www.thesun.ie': 4.481550524994165e-05,\n",
+       " 'www.dallasnews.com': 4.481432357267996e-05,\n",
+       " 'c-s-s-a.org': 4.4780953876756535e-05,\n",
+       " 'caribbean.winne.com': 4.4775538460103305e-05,\n",
+       " 'cherryblossomtreeimages.netlify.app': 4.47707936271722e-05,\n",
+       " 'www.wisdomlib.org': 4.4762260902405335e-05,\n",
+       " 'hv.greenspun.com': 4.475602991095494e-05,\n",
+       " 'www.no2nuclearpower.org.uk': 4.4755558689961015e-05,\n",
+       " 'slife.org': 4.459137804612485e-05,\n",
+       " 'motm.kicks-ass.net': 4.454331712952168e-05,\n",
+       " 'thenewsnigeria.com.ng': 4.453633218448101e-05,\n",
+       " 'www.marketwatch.com': 4.449777905762444e-05,\n",
+       " 'hotair.com': 4.445648197467249e-05,\n",
+       " 'www.sb.marketwatch.com': 4.438000643213583e-05,\n",
+       " 'lolcow.farm': 4.427683078312817e-05,\n",
+       " 'coasterbuzz.com': 4.418396399955678e-05,\n",
+       " 'staging.pastemagazine.com': 4.415840932257871e-05,\n",
+       " 'english.newsnationtv.com': 4.413182158419087e-05,\n",
+       " 'www.playbill.com': 4.412994032499206e-05,\n",
+       " 'www.kmuw.org': 4.410439289756774e-05,\n",
+       " 'db-engines.com': 4.409923121529586e-05,\n",
+       " 'comicbook.com': 4.408087534519416e-05,\n",
+       " 'original.newsbreak.com': 4.407597464685737e-05,\n",
+       " 'www.courierpostonline.com': 4.397977306855976e-05,\n",
+       " 'www.lovewhatmatters.com': 4.3963660935344505e-05,\n",
+       " 'www.9news.com.au': 4.39539102855472e-05,\n",
+       " 'www.emirates247.com': 4.391295030684475e-05,\n",
+       " 'www.ipsnews.net': 4.388738475553604e-05,\n",
+       " 'sparklefaerydreams.com': 4.38864568126557e-05,\n",
+       " 'www.windowscentral.com': 4.3863279989308495e-05,\n",
+       " 'grantome.com': 4.3831160841407393e-05,\n",
+       " 'boards.weddingbee.com': 4.3814976212654615e-05,\n",
+       " 'www.fda.gov': 4.379600413048394e-05,\n",
+       " 'www.trulia.com': 4.379217274132567e-05,\n",
+       " 'windowsreport.com': 4.37904980944088e-05,\n",
+       " 'news.wosu.org': 4.378937078880027e-05,\n",
+       " 'www.belfastlive.co.uk': 4.377158763344503e-05,\n",
+       " '0-bmcinfectdis-biomedcentral-com.brum.beds.ac.uk': 4.369082760464056e-05,\n",
+       " 'www.dispatch.com': 4.364714179372711e-05,\n",
+       " 'pagancentral.org': 4.359199081355386e-05,\n",
+       " 'm.egwwritings.org': 4.354498833179858e-05,\n",
+       " 'fivethirtyeight.com': 4.35326604656422e-05,\n",
+       " 'www.theautomaticearth.com': 4.351979613250813e-05,\n",
+       " 'www.coventrytelegraph.net': 4.350388336202108e-05,\n",
+       " 'www.lawgazette.co.uk': 4.344794218048879e-05,\n",
+       " 'www.ozy.com': 4.343802116617829e-05,\n",
+       " 'www.glamour.com': 4.3415333687709385e-05,\n",
+       " 'www.idownloadblog.com': 4.340395551309461e-05,\n",
+       " 'apps.shopify.com': 4.3145092072472e-05,\n",
+       " 'www.sheknows.com': 4.31031461544592e-05,\n",
+       " 'www.dailystar.co.uk': 4.309298590487486e-05,\n",
+       " 'www.thrillist.com': 4.305201142706491e-05,\n",
+       " 'athlonsports.com': 4.299827410987342e-05,\n",
+       " 'www.reddit.com': 4.295538212509589e-05,\n",
+       " 'es.uhaul.com': 4.2948872025826016e-05,\n",
+       " 'bmcpsychiatry.biomedcentral.com': 4.2947364118645467e-05,\n",
+       " 'www.thestarpress.com': 4.290172092821881e-05,\n",
+       " 'sportslens.com': 4.285106829614907e-05,\n",
+       " 'lists.advaita-vedanta.org': 4.268204857518301e-05,\n",
+       " 'www.mixonline.com': 4.264579355686603e-05,\n",
+       " 'hstreasures.com': 4.262433850253508e-05,\n",
+       " 'www.technologynetworks.com': 4.262179028439102e-05,\n",
+       " 'bangordailynews.com': 4.2552988394501534e-05,\n",
+       " 'www.complex.com': 4.2549657224552194e-05,\n",
+       " 'virologyj.biomedcentral.com': 4.25204306485984e-05,\n",
+       " 'news.bbc.co.uk': 4.25185167664077e-05,\n",
+       " 'committees.parliament.uk': 4.250431851538315e-05,\n",
+       " 'local.job-applications.com': 4.243543688040238e-05,\n",
+       " 'www.mondaq.com': 4.239263189026988e-05,\n",
+       " 'www.newsday.co.zw': 4.235220837854513e-05,\n",
+       " 'www.independent.com': 4.231493117314903e-05,\n",
+       " 'financialpost.com': 4.2311987854325456e-05,\n",
+       " 'www.nydailynews.com': 4.229718064078568e-05,\n",
+       " 'www.lincolnshirelive.co.uk': 4.228084014662721e-05,\n",
+       " 'www.nairaland.com': 4.22537449394767e-05,\n",
+       " 'www.weforum.org': 4.224628514866522e-05,\n",
+       " 'www.ishn.com': 4.224242838606882e-05,\n",
+       " 'phenomenica.com': 4.21905324555305e-05,\n",
+       " 'collegebasketball.nbcsports.com': 4.212842552853158e-05,\n",
+       " 'ncf.sobek.ufl.edu': 4.2059971617222224e-05,\n",
+       " 'www.euractiv.com': 4.205985199958531e-05,\n",
+       " 'www.ocregister.com': 4.204707103631939e-05,\n",
+       " 'podcasts.apple.com': 4.199048464450312e-05,\n",
+       " 'www.universetoday.com': 4.196350543021264e-05,\n",
+       " 'www.sec.marketwatch.com': 4.19586446044215e-05,\n",
+       " 'www.spokesman.com': 4.193551852795056e-05,\n",
+       " 'www.fishbowlapp.com': 4.186291062234093e-05,\n",
+       " 'cointelegraph.com': 4.185440689578908e-05,\n",
+       " 'bluray.highdefdigest.com': 4.172987406142613e-05,\n",
+       " 'www.bishop-accountability.org': 4.171900698035091e-05,\n",
+       " 'wilearncap.asuscomm.com': 4.169513782462033e-05,\n",
+       " 'www.gofundme.com': 4.166943090701344e-05,\n",
+       " 'library.kiwix.org': 4.166776350965033e-05,\n",
+       " 'www.laptopmag.com': 4.166093805479222e-05,\n",
+       " 'wgday.net': 4.163025431853415e-05,\n",
+       " 'punchng.com': 4.1627372620917465e-05,\n",
+       " 'www.vanguardngr.com': 4.162552760948742e-05,\n",
+       " 'www.oregonlive.com': 4.159901236663712e-05,\n",
+       " 'philly.metro.us': 4.15494145446384e-05,\n",
+       " 'elifesciences.org': 4.149664141809601e-05,\n",
+       " 'www.kirkusreviews.com': 4.14384673740079e-05,\n",
+       " 'pmg.org.za': 4.143711895700991e-05,\n",
+       " 'www.casino.org': 4.1405554399970885e-05,\n",
+       " 'www.theblaze.com': 4.134392594351965e-05,\n",
+       " 'nonprofitquarterly.org': 4.131678361426974e-05,\n",
+       " 'www.cbsnews.com': 4.127287669196684e-05,\n",
+       " 'tvline.com': 4.1269871751936365e-05,\n",
+       " 'ftw.usatoday.com': 4.1261828372047806e-05,\n",
+       " 'www.presidency.ucsb.edu': 4.124738001141878e-05,\n",
+       " 'www.news24.com': 4.1196886869531606e-05,\n",
+       " 'www.mtv.com': 4.119597705053565e-05,\n",
+       " 'freshairarchive.org': 4.119180855712788e-05,\n",
+       " 'osaa.dk': 4.113290230811076e-05,\n",
+       " 'bleedingcool.com': 4.106717785378927e-05,\n",
+       " 'www.brainscape.com': 4.103747643206468e-05,\n",
+       " 'www.sbs.com.au': 4.101813099787574e-05,\n",
+       " 'forums.theregister.com': 4.093041864702248e-05,\n",
+       " 'www.cleveland.com': 4.088752666224495e-05,\n",
+       " 'kotaku.com': 4.085596572998281e-05,\n",
+       " 'www.definitions.net': 4.0845301636412665e-05,\n",
+       " 'www.evwind.es': 4.081171082909978e-05,\n",
+       " 'www.wind-watch.org': 4.079068712321711e-05,\n",
+       " 'www.pressconnects.com': 4.078646063337932e-05,\n",
+       " 'www.nfl.com': 4.07140810887129e-05,\n",
+       " 'pike.lysator.liu.se': 4.071288491234371e-05,\n",
+       " 'www.getreading.co.uk': 4.070931088234366e-05,\n",
+       " 'smallbiztrends.com': 4.07008071557918e-05,\n",
+       " 'www.techtimes.com': 4.068088175730266e-05,\n",
+       " 'blog.hubspot.com': 4.066579181116653e-05,\n",
+       " 'practicaldev-herokuapp-com.global.ssl.fastly.net': 4.0643832912849765e-05,\n",
+       " 'www.wbur.org': 4.060892993630765e-05,\n",
+       " 'fcw.com': 4.059569950070907e-05,\n",
+       " 'bmjopen.bmj.com': 4.055734936135758e-05,\n",
+       " 'asia.nikkei.com': 4.054750446736148e-05,\n",
+       " 'www.flsenate.gov': 4.0544376284917216e-05,\n",
+       " 'www.thespruceeats.com': 4.051689685141781e-05,\n",
+       " 'www.cambstimes.co.uk': 4.0463540135798334e-05,\n",
+       " 'efinne1540.wordpress.com': 4.0440820034337546e-05,\n",
+       " 'mb.com.ph': 4.0383961784255546e-05,\n",
+       " 'www.pnj.com': 4.03558298909184e-05,\n",
+       " 'friendlyatheist.patheos.com': 4.033316778588763e-05,\n",
+       " 'www.jayski.com': 4.030019319064372e-05,\n",
+       " 'www.un.org': 4.029648866867612e-05,\n",
+       " 'www.goodhousekeeping.com': 4.028661477646501e-05,\n",
+       " 'www.americanprogress.org': 4.0260081409730334e-05,\n",
+       " 'www.thisisanfield.com': 4.021005948883708e-05,\n",
+       " 'en-academic.com': 4.017458742232538e-05,\n",
+       " 'www.ecowatch.com': 4.011691722222308e-05,\n",
+       " 'm.orlandoweekly.com': 4.0110331002638805e-05,\n",
+       " 'www.rogerebert.com': 4.0106738848754366e-05,\n",
+       " 'www.cbpp.org': 4.007380050127922e-05,\n",
+       " 'theinventory.com': 4.006688805177608e-05,\n",
+       " 'www.jmir.org': 4.005055843194824e-05,\n",
+       " 'www.irs.gov': 4.0033590851390175e-05,\n",
+       " 'books.google.gr': 4.003322837370254e-05,\n",
+       " 'simple.wikipedia.org': 4.001320873101461e-05,\n",
+       " 'www.smartcompany.com.au': 3.999999279452353e-05,\n",
+       " 'www.boxinginsider.com': 3.9964658469533137e-05,\n",
+       " 'electrek.co': 3.996420899720047e-05,\n",
+       " 'www.earlynewspaper.com': 3.995206236988791e-05,\n",
+       " 'visionbib.com': 3.994921692004e-05,\n",
+       " 'www.oom2.com': 3.9939150914654456e-05,\n",
+       " 'www.pilotonline.com': 3.989394269745296e-05,\n",
+       " 'oilprice.com': 3.988750871849748e-05,\n",
+       " 'www.androidcentral.com': 3.98828218819964e-05,\n",
+       " 'everything2.com': 3.988159308263532e-05,\n",
+       " 'www.tvtechnology.com': 3.981694881182298e-05,\n",
+       " 'jalopnik.com': 3.971168891611144e-05,\n",
+       " 'fr.uhaul.com': 3.970924219171992e-05,\n",
+       " 'wn.com': 3.964988284559325e-05,\n",
+       " 'laist.com': 3.963267965453823e-05,\n",
+       " 'www.kirchgemeinde-roeschenz.ch': 3.961209817143447e-05,\n",
+       " 'www.ydr.com': 3.959250987719482e-05,\n",
+       " 'www.vogue.com': 3.9586195515876265e-05,\n",
+       " 'motorsports.nbcsports.com': 3.958328844482145e-05,\n",
+       " 'www.coinspeaker.com': 3.956746991853318e-05,\n",
+       " 'www.essence.com': 3.956449035194085e-05,\n",
+       " 'www.rsvplive.ie': 3.955596850150461e-05,\n",
+       " 'www.epo.org': 3.948293649700045e-05,\n",
+       " 'forums.cfl.ca': 3.946960094287246e-05,\n",
+       " 'spacecoastdaily.com': 3.9415624390407136e-05,\n",
+       " 'britishlistedbuildings.co.uk': 3.9370361801552494e-05,\n",
+       " 'books.google.co.kr': 3.932360942940168e-05,\n",
+       " 'phys.org': 3.932302221554772e-05,\n",
+       " 'www.outsidethebeltway.com': 3.931450036511148e-05,\n",
+       " 'people.com': 3.931203551683558e-05,\n",
+       " 'www.jta.org': 3.930082770673399e-05,\n",
+       " 'sg.hotels.com': 3.9292255109421485e-05,\n",
+       " 'ucanr.edu': 3.929047896875209e-05,\n",
+       " 'www.revisor.mn.gov': 3.927835409010079e-05,\n",
+       " 'english.alarabiya.net': 3.92559094716826e-05,\n",
+       " 'www.marketplace.org': 3.920590567467372e-05,\n",
+       " 'wearethemighty.rebelmouse.com': 3.919585416839568e-05,\n",
+       " '1library.net': 3.918777091596148e-05,\n",
+       " 'www.mansfieldnewsjournal.com': 3.918767667176269e-05,\n",
+       " 'www.aging-us.com': 3.917408738325336e-05,\n",
+       " 'diehardgamefan.com': 3.916376401870959e-05,\n",
+       " 'www.rhsupplies.org': 3.914253732532184e-05,\n",
+       " 'erenow.net': 3.9135585003273055e-05,\n",
+       " 'owlcation.com': 3.913422208716756e-05,\n",
+       " 'www.gamedeveloper.com': 3.911479690788735e-05,\n",
+       " 'www.idcrawl.com': 3.904242461277468e-05,\n",
+       " 'carnegieendowment.org': 3.89848740303093e-05,\n",
+       " 'www.thedailyjournal.com': 3.896226629693168e-05,\n",
+       " 'webot.org': 3.8944465017692056e-05,\n",
+       " 'content.yudu.com': 3.892349568346253e-05,\n",
+       " 'www.colts.com': 3.892286497228605e-05,\n",
+       " 'access.redhat.com': 3.891852611436509e-05,\n",
+       " 'wowpedia.fandom.com': 3.890621274731622e-05,\n",
+       " 'www.drf.com': 3.886028319951633e-05,\n",
+       " 'en.m.wikibooks.org': 3.8852707415844816e-05,\n",
+       " 'support.mozilla.org': 3.882986769674711e-05,\n",
+       " 'www.monexsecurities.com.au': 3.8821490837385927e-05,\n",
+       " 'blog.flibo.ai': 3.876277670154324e-05,\n",
+       " 'books.google.com.eg': 3.876086281935255e-05,\n",
+       " 'lists.gnupg.org': 3.873693929196881e-05,\n",
+       " 'www.miastenia.it': 3.87040118188243e-05,\n",
+       " 'www.wuwm.com': 3.870314187237398e-05,\n",
+       " 'legaltalknetwork.com': 3.8688117172221625e-05,\n",
+       " 'books.google.no': 3.862479956974601e-05,\n",
+       " 'marketbusinessnews.com': 3.860286966964426e-05,\n",
+       " 'epjournal.net': 3.856268176841647e-05,\n",
+       " 'www.mlbtraderumors.com': 3.85441374099172e-05,\n",
+       " 'arstechnica.com': 3.854348495007946e-05,\n",
+       " 'blueandgreentomorrow.com': 3.851208350799988e-05,\n",
+       " 'www.lawnet.gov.lk': 3.849381463254321e-05,\n",
+       " 'lawofselfdefense.com': 3.8400063403414e-05,\n",
+       " 'www.rd.com': 3.838551717380931e-05,\n",
+       " 'www.outdoorlife.com': 3.833586498015744e-05,\n",
+       " 'www.packers.com': 3.833264617829127e-05,\n",
+       " 'www.revolt.tv': 3.831813982123222e-05,\n",
+       " 'www.firstshowing.net': 3.828763732381796e-05,\n",
+       " 'www.wesa.fm': 3.8286252659051205e-05,\n",
+       " 'www.ammoland.com': 3.82664287543146e-05,\n",
+       " 'monetmagazine.top': 3.826308308525775e-05,\n",
+       " 'www.voanews.com': 3.826214064326991e-05,\n",
+       " 'mg.co.za': 3.824225149254952e-05,\n",
+       " 'stemcellres.biomedcentral.com': 3.82395147860079e-05,\n",
+       " 'community.atlassian.com': 3.8226657702427584e-05,\n",
+       " 'www.counselling-directory.org.uk': 3.822351864565269e-05,\n",
+       " 'www.nationalreview.com': 3.817521849377568e-05,\n",
+       " 'www.webmd.com': 3.8168472784008847e-05,\n",
+       " 'www.portsmouth.co.uk': 3.815725409957663e-05,\n",
+       " 'www.montgomeryadvertiser.com': 3.813558155863309e-05,\n",
+       " 'www.blogarama.com': 3.812341680743616e-05,\n",
+       " 'www.usmagazine.com': 3.810199437609708e-05,\n",
+       " 'www.diverseeducation.com': 3.809790200300371e-05,\n",
+       " 'www.lexology.com': 3.807703053774985e-05,\n",
+       " 'www.modernhealthcare.com': 3.8055782095700845e-05,\n",
+       " 'amt.copernicus.org': 3.803882901425029e-05,\n",
+       " 'leadership.ng': 3.798082895945224e-05,\n",
+       " 'www.imore.com': 3.797297044318437e-05,\n",
+       " 'electricliterature.com': 3.796756952563865e-05,\n",
+       " 'community.spiceworks.com': 3.792525025560758e-05,\n",
+       " 'www.consumeraffairs.com': 3.784720880946035e-05,\n",
+       " 'adops.motherjones.com': 3.7835029559155904e-05,\n",
+       " 'www.cbssports.com': 3.778831705955073e-05,\n",
+       " 'www.mail-archive.com': 3.778817206847568e-05,\n",
+       " 'www.tumbral.com': 3.7780146812471496e-05,\n",
+       " 'www.thestar.co.uk': 3.7742706492115954e-05,\n",
+       " 'www.tampabay.com': 3.77381682714668e-05,\n",
+       " 'yourmagazine.top': 3.773286159811987e-05,\n",
+       " 'www.freeadvice.com': 3.772870397904272e-05,\n",
+       " 'www.macleans.ca': 3.77257824088804e-05,\n",
+       " ...}"
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 16
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-15T14:01:34.100514Z",
+     "start_time": "2024-05-15T14:01:34.096130Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "filtered_df = df_43[df_43['in_49'] == 0]  # Filter rows where 'in_49' is 0\n",
+    "sorted_df = filtered_df.sort_values(by='Frequency', ascending=False)  # Sort by 'Frequency' column in descending order"
+   ],
+   "id": "274edf9d4064ad1d",
+   "outputs": [],
+   "execution_count": 33
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-15T14:01:36.920220Z",
+     "start_time": "2024-05-15T14:01:36.914063Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "sorted_df",
+   "id": "7d62dfa545a519f1",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "                              URL  Frequency  in_49  change_to_49\n",
+       "9                    ufdc.ufl.edu   0.000443    0.0     -0.000443\n",
+       "22           www.hotfreebooks.com   0.000244    0.0     -0.000244\n",
+       "37             irclogs.ubuntu.com   0.000190    0.0     -0.000190\n",
+       "47     transparentpng.netlify.app   0.000181    0.0     -0.000181\n",
+       "85          www.preceptaustin.org   0.000120    0.0     -0.000120\n",
+       "...                           ...        ...    ...           ...\n",
+       "59994      www.annahelizabeth.com   0.000001    0.0     -0.000001\n",
+       "59996              meisendorf.com   0.000001    0.0     -0.000001\n",
+       "59997        www.anyrubbish.co.uk   0.000001    0.0     -0.000001\n",
+       "59998                  qjshhxx.cn   0.000001    0.0     -0.000001\n",
+       "59999       www.al-enterprise.com   0.000001    0.0     -0.000001\n",
+       "\n",
+       "[29485 rows x 4 columns]"
+      ],
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>URL</th>\n",
+       "      <th>Frequency</th>\n",
+       "      <th>in_49</th>\n",
+       "      <th>change_to_49</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
+       "      <td>ufdc.ufl.edu</td>\n",
+       "      <td>0.000443</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>-0.000443</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>22</th>\n",
+       "      <td>www.hotfreebooks.com</td>\n",
+       "      <td>0.000244</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>-0.000244</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>37</th>\n",
+       "      <td>irclogs.ubuntu.com</td>\n",
+       "      <td>0.000190</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>-0.000190</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>47</th>\n",
+       "      <td>transparentpng.netlify.app</td>\n",
+       "      <td>0.000181</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>-0.000181</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>85</th>\n",
+       "      <td>www.preceptaustin.org</td>\n",
+       "      <td>0.000120</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>-0.000120</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>59994</th>\n",
+       "      <td>www.annahelizabeth.com</td>\n",
+       "      <td>0.000001</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>-0.000001</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>59996</th>\n",
+       "      <td>meisendorf.com</td>\n",
+       "      <td>0.000001</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>-0.000001</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>59997</th>\n",
+       "      <td>www.anyrubbish.co.uk</td>\n",
+       "      <td>0.000001</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>-0.000001</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>59998</th>\n",
+       "      <td>qjshhxx.cn</td>\n",
+       "      <td>0.000001</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>-0.000001</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>59999</th>\n",
+       "      <td>www.al-enterprise.com</td>\n",
+       "      <td>0.000001</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>-0.000001</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>29485 rows × 4 columns</p>\n",
+       "</div>"
+      ]
+     },
+     "execution_count": 34,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 34
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-15T14:03:08.719028Z",
+     "start_time": "2024-05-15T14:03:08.082516Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "assert all(row[1][\"URL\"] not in freqs_49 for row in sorted_df.iterrows())",
+   "id": "3a2317033c481119",
+   "outputs": [],
+   "execution_count": 36
+  },
+  {
+   "metadata": {},
+   "cell_type": "code",
+   "outputs": [],
+   "execution_count": null,
+   "source": "",
+   "id": "d67bd99d6e230caf"
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/loubna-ablations_faq_metrics.csv

@@ -0,0 +1,27 @@
+runname,seed,steps,agg_score,commonsense_qa/acc,commonsense_qa/acc_norm,hellaswag/acc,hellaswag/acc_norm,openbookqa/acc,openbookqa/acc_norm,piqa/acc,piqa/acc_norm,siqa/acc,siqa/acc_norm,winogrande/acc,winogrande/acc_norm,all/acc,all/acc_norm,arc/acc,arc/acc_norm,mmlu/acc,mmlu/acc_norm
+filtered_web_min_score_4_fix-seed-1,0,1000,0.3682507313787937,0.2540000081062317,0.2709999978542328,0.2800000011920929,0.2840000092983246,0.13600000739097595,0.3059999942779541,0.5759999752044678,0.5789999961853027,0.35100001096725464,0.38100001215934753,0.4909999966621399,0.503000020980835,0.26319819688796997,0.2825128138065338,0.34049999713897705,0.3529999852180481,0.2515593469142914,0.26900583505630493
+filtered_web_min_score_4_fix-seed-1,0,2000,0.38916464149951935,0.2980000078678131,0.27900001406669617,0.3009999990463257,0.3240000009536743,0.18799999356269836,0.32600000500679016,0.6010000109672546,0.6079999804496765,0.36000001430511475,0.37599998712539673,0.5019999742507935,0.503000020980835,0.2754783630371094,0.2941242456436157,0.41100001335144043,0.4189999997615814,0.26024726033210754,0.2783171236515045
+filtered_web_min_score_4_fix-seed-1,0,3000,0.39035574346780777,0.2840000092983246,0.27900001406669617,0.3050000071525574,0.3269999921321869,0.19200000166893005,0.32199999690055847,0.6150000095367432,0.6200000047683716,0.3619999885559082,0.38199999928474426,0.5149999856948853,0.49000000953674316,0.2794281244277954,0.2956494987010956,0.4259999990463257,0.4230000078678131,0.263821542263031,0.2798459231853485
+filtered_web_min_score_4_fix-seed-1,0,4000,0.4036811888217926,0.3109999895095825,0.28600001335144043,0.31299999356269836,0.3440000116825104,0.21199999749660492,0.3240000009536743,0.6200000047683716,0.6439999938011169,0.35600000619888306,0.38199999928474426,0.5120000243186951,0.5019999742507935,0.29403528571128845,0.3073711395263672,0.44749999046325684,0.45649999380111694,0.2788296937942505,0.2909495234489441
+filtered_web_min_score_4_fix-seed-1,0,5000,0.4085462875664234,0.3050000071525574,0.2930000126361847,0.32499998807907104,0.35499998927116394,0.22200000286102295,0.3540000021457672,0.628000020980835,0.6359999775886536,0.36399999260902405,0.3880000114440918,0.5180000066757202,0.49799999594688416,0.2961043119430542,0.31081706285476685,0.4544999897480011,0.44999998807907104,0.2802768647670746,0.2943703234195709
+filtered_web_min_score_4_fix-seed-1,0,6000,0.41010819375514984,0.30000001192092896,0.289000004529953,0.3330000042915344,0.35600000619888306,0.20600000023841858,0.3440000116825104,0.6259999871253967,0.640999972820282,0.3720000088214874,0.38199999928474426,0.5379999876022339,0.5019999742507935,0.30340614914894104,0.3206590712070465,0.4625000059604645,0.46149998903274536,0.28809472918510437,0.30536559224128723
+filtered_web_min_score_4_fix-seed-1,0,7000,0.4124617166817188,0.30399999022483826,0.2849999964237213,0.33799999952316284,0.36800000071525574,0.2199999988079071,0.3479999899864197,0.6269999742507935,0.6380000114440918,0.37299999594688416,0.382999986410141,0.5389999747276306,0.5049999952316284,0.30451327562332153,0.3200468420982361,0.47099998593330383,0.4684999883174896,0.2886028587818146,0.30419376492500305
+filtered_web_min_score_4_fix-seed-1,0,8000,0.41384265199303627,0.3199999928474426,0.2840000092983246,0.3330000042915344,0.3720000088214874,0.21400000154972076,0.3540000021457672,0.6309999823570251,0.6399999856948853,0.36500000953674316,0.3779999911785126,0.5370000004768372,0.5080000162124634,0.30659323930740356,0.32282689213752747,0.4650000035762787,0.4675000011920929,0.2912028133869171,0.30724120140075684
+fineweb_2B_educational_minimum_score_3-seed-0,0,1000,0.3635076731443405,0.2549999952316284,0.25999999046325684,0.2809999883174896,0.2849999964237213,0.1420000046491623,0.2759999930858612,0.5899999737739563,0.6209999918937683,0.36500000953674316,0.37599998712539673,0.503000020980835,0.48899999260902405,0.26151496171951294,0.27996155619621277,0.31299999356269836,0.33399999141693115,0.2497626692056656,0.2670614421367645
+fineweb_2B_educational_minimum_score_3-seed-0,0,2000,0.383779339492321,0.28299999237060547,0.2800000011920929,0.3089999854564667,0.3330000042915344,0.164000004529953,0.3100000023841858,0.6230000257492065,0.6399999856948853,0.3580000102519989,0.3709999918937683,0.5040000081062317,0.49399998784065247,0.26660147309303284,0.28831353783607483,0.3634999990463257,0.36899998784065247,0.2519490420818329,0.2732347548007965
+fineweb_2B_educational_minimum_score_3-seed-0,0,4000,0.4050140306353569,0.3089999854564667,0.29499998688697815,0.34299999475479126,0.3790000081062317,0.20800000429153442,0.328000009059906,0.6570000052452087,0.675000011920929,0.36399999260902405,0.3840000033378601,0.5080000162124634,0.4950000047683716,0.28317055106163025,0.3020445704460144,0.40049999952316284,0.398499995470047,0.2669488787651062,0.28561222553253174
+fineweb_2B_educational_minimum_score_3-seed-0,0,6000,0.41626306250691414,0.33399999141693115,0.3140000104904175,0.35600000619888306,0.4189999997615814,0.21400000154972076,0.3400000035762787,0.6650000214576721,0.6800000071525574,0.3580000102519989,0.3880000114440918,0.5040000081062317,0.4869999885559082,0.2945695221424103,0.31046855449676514,0.421999990940094,0.40849998593330383,0.278456449508667,0.2936044931411743
+fineweb_2B_educational_minimum_score_3-seed-0,0,8000,0.42167554423213005,0.33899998664855957,0.3140000104904175,0.3659999966621399,0.4390000104904175,0.21400000154972076,0.3400000035762787,0.6489999890327454,0.6819999814033508,0.3709999918937683,0.38999998569488525,0.4970000088214874,0.4869999885559082,0.3004492521286011,0.31168535351753235,0.4350000023841858,0.42750000953674316,0.28461754322052,0.2939043641090393
+fineweb_2B_educational_regression-seed-6,0,1000,0.3628821112215519,0.24899999797344208,0.25999999046325684,0.28700000047683716,0.30300000309944153,0.1379999965429306,0.2720000147819519,0.6029999852180481,0.625,0.36000001430511475,0.38499999046325684,0.5120000243186951,0.4970000088214874,0.2574775218963623,0.2720729410648346,0.29350000619888306,0.30250000953674316,0.24561470746994019,0.25855687260627747
+fineweb_2B_educational_regression-seed-6,0,2000,0.3791401833295822,0.27900001406669617,0.2639999985694885,0.3149999976158142,0.3230000138282776,0.15800000727176666,0.2919999957084656,0.6299999952316284,0.6520000100135803,0.3569999933242798,0.3880000114440918,0.5180000066757202,0.4970000088214874,0.26788005232810974,0.28260648250579834,0.3255000114440918,0.34950000047683716,0.2544597089290619,0.2676214277744293
+fineweb_2B_educational_regression-seed-6,0,4000,0.4023119993507862,0.3190000057220459,0.3050000071525574,0.3490000069141388,0.38600000739097595,0.17399999499320984,0.3179999887943268,0.6549999713897705,0.6800000071525574,0.36399999260902405,0.382999986410141,0.5080000162124634,0.5,0.27732229232788086,0.29341956973075867,0.37049999833106995,0.3700000047683716,0.26168331503868103,0.2764959931373596
+fineweb_2B_educational_regression-seed-6,0,6000,0.41444139182567596,0.3310000002384186,0.3149999976158142,0.3580000102519989,0.4180000126361847,0.1860000044107437,0.33799999952316284,0.6729999780654907,0.6990000009536743,0.3700000047683716,0.3790000081062317,0.5199999809265137,0.4950000047683716,0.28314903378486633,0.29828882217407227,0.3855000138282776,0.39149999618530273,0.26659101247787476,0.28003111481666565
+fineweb_2B_educational_regression-seed-6,0,8000,0.41904011741280556,0.31299999356269836,0.31700000166893005,0.36800000071525574,0.421999990940094,0.18000000715255737,0.3499999940395355,0.6909999847412109,0.6970000267028809,0.3630000054836273,0.3799999952316284,0.5329999923706055,0.5139999985694885,0.28121474385261536,0.30207374691963196,0.3889999985694885,0.38850000500679016,0.2640869915485382,0.2838209271430969
+fineweb_2024_10_all_2B-seed-6,0,1000,0.35314396768808365,0.2199999988079071,0.24300000071525574,0.2919999957084656,0.30799999833106995,0.1340000033378601,0.257999986410141,0.5799999833106995,0.5870000123977661,0.3529999852180481,0.38499999046325684,0.4909999966621399,0.4869999885559082,0.2508137822151184,0.2687792479991913,0.27649998664855957,0.3009999990463257,0.23999817669391632,0.2561517655849457
+fineweb_2024_10_all_2B-seed-6,0,2000,0.37468964606523514,0.2759999930858612,0.2759999930858612,0.3050000071525574,0.32499998807907104,0.13199999928474426,0.2720000147819519,0.628000020980835,0.6520000100135803,0.3540000021457672,0.3889999985694885,0.5040000081062317,0.49799999594688416,0.2582138776779175,0.27513813972473145,0.3070000112056732,0.3255000114440918,0.24510353803634644,0.26001715660095215
+fineweb_2024_10_all_2B-seed-6,0,3000,0.38586894795298576,0.2669999897480011,0.2809999883174896,0.3269999921321869,0.35899999737739563,0.17800000309944153,0.2879999876022339,0.6420000195503235,0.6769999861717224,0.3709999918937683,0.3880000114440918,0.5099999904632568,0.49799999594688416,0.26085224747657776,0.27577292919158936,0.32249999046325684,0.3370000123977661,0.24588415026664734,0.2589516043663025
+fineweb_2024_10_all_2B-seed-6,0,4000,0.38998349756002426,0.2750000059604645,0.2809999883174896,0.35199999809265137,0.382999986410141,0.15199999511241913,0.28600001335144043,0.6480000019073486,0.6840000152587891,0.36500000953674316,0.38499999046325684,0.5049999952316284,0.49300000071525574,0.26505371928215027,0.2810457646846771,0.3264999985694885,0.3434999883174896,0.2504998445510864,0.2643679976463318
+fineweb_2024_10_all_2B-seed-6,0,5000,0.3979869969189167,0.30300000309944153,0.296999990940094,0.3490000069141388,0.3970000147819519,0.15399999916553497,0.28999999165534973,0.6690000295639038,0.6940000057220459,0.375,0.382999986410141,0.5090000033378601,0.5019999742507935,0.26854822039604187,0.2826780378818512,0.33399999141693115,0.35600000619888306,0.25313395261764526,0.2648960053920746
+fineweb_2024_10_all_2B-seed-6,0,6000,0.4039541743695736,0.31700000166893005,0.3190000057220459,0.35899999737739563,0.41600000858306885,0.16599999368190765,0.2840000092983246,0.6600000262260437,0.6949999928474426,0.3790000081062317,0.4000000059604645,0.515999972820282,0.49000000953674316,0.26819688081741333,0.2866784930229187,0.3330000042915344,0.35899999737739563,0.25210171937942505,0.2686333656311035
+fineweb_2024_10_all_2B-seed-6,0,7000,0.4048592820763588,0.2980000078678131,0.3100000023841858,0.367000013589859,0.42399999499320984,0.17599999904632568,0.28999999165534973,0.6729999780654907,0.7020000219345093,0.38199999928474426,0.3959999978542328,0.5109999775886536,0.49399998784065247,0.27170079946517944,0.2894589602947235,0.32499998807907104,0.35100001096725464,0.25620266795158386,0.27187424898147583
+fineweb_2024_10_all_2B-seed-6,0,8000,0.40328324213624,0.33000001311302185,0.3190000057220459,0.36399999260902405,0.41200000047683716,0.17599999904632568,0.2759999930858612,0.6579999923706055,0.703000009059906,0.382999986410141,0.40299999713897705,0.5099999904632568,0.49300000071525574,0.2675325274467468,0.2870177924633026,0.3294999897480011,0.35100001096725464,0.2510458528995514,0.26926591992378235

notebooks/loubna-edu_fw_ablations_metrics.csv

@@ -0,0 +1,83 @@
+runname,seed,steps,agg_score,commonsense_qa/acc,commonsense_qa/acc_norm,hellaswag/acc,hellaswag/acc_norm,openbookqa/acc,openbookqa/acc_norm,piqa/acc,piqa/acc_norm,siqa/acc,siqa/acc_norm,winogrande/acc,winogrande/acc_norm,all/acc,all/acc_norm,arc/acc,arc/acc_norm,mmlu/acc,mmlu/acc_norm
+edu_fineweb_350b_tokens-seed-1,0,2000,0.3903256542980671,0.2840000092983246,0.28299999237060547,0.3140000104904175,0.32499998807907104,0.164000004529953,0.29600000381469727,0.6230000257492065,0.6320000290870667,0.3619999885559082,0.4059999883174896,0.5109999775886536,0.5109999775886536,0.2796742022037506,0.29916155338287354,0.37950000166893005,0.38499999046325684,0.26599687337875366,0.2846052646636963
+edu_fineweb_350b_tokens-seed-1,0,4000,0.4146799184381962,0.32199999690055847,0.3070000112056732,0.34299999475479126,0.39500001072883606,0.19599999487400055,0.3199999928474426,0.656000018119812,0.6880000233650208,0.3709999918937683,0.3880000114440918,0.5180000066757202,0.4950000047683716,0.29061299562454224,0.31259292364120483,0.42149999737739563,0.4284999966621399,0.2744007706642151,0.29593929648399353
+edu_fineweb_350b_tokens-seed-1,0,6000,0.42839035764336586,0.3190000057220459,0.3109999895095825,0.3720000088214874,0.4309999942779541,0.20200000703334808,0.35199999809265137,0.6600000262260437,0.6700000166893005,0.37299999594688416,0.3919999897480011,0.5199999809265137,0.5189999938011169,0.303979754447937,0.323323130607605,0.43149998784065247,0.44600000977516174,0.2885909378528595,0.30612286925315857
+edu_fineweb_350b_tokens-seed-1,0,8000,0.4436151087284088,0.3400000035762787,0.3109999895095825,0.3790000081062317,0.46299999952316284,0.20399999618530273,0.36000001430511475,0.6809999942779541,0.699999988079071,0.3840000033378601,0.40400001406669617,0.5170000195503235,0.5170000195503235,0.3151480555534363,0.3332844376564026,0.46299999952316284,0.4790000021457672,0.2991863787174225,0.3149208426475525
+edu_fineweb_350b_tokens-seed-1,0,10000,0.4414566531777382,0.34599998593330383,0.31700000166893005,0.38999998569488525,0.45399999618530273,0.22200000286102295,0.36399999260902405,0.6899999976158142,0.6959999799728394,0.3659999966621399,0.39500001072883606,0.5139999985694885,0.5059999823570251,0.3189352750778198,0.33541902899742126,0.48899999260902405,0.4819999933242798,0.30218935012817383,0.31765326857566833
+edu_fineweb_350b_tokens-seed-1,0,12000,0.4507521316409111,0.36800000071525574,0.33500000834465027,0.39399999380111694,0.4699999988079071,0.23600000143051147,0.3659999966621399,0.6959999799728394,0.7020000219345093,0.3700000047683716,0.39899998903274536,0.5389999747276306,0.5320000052452087,0.32298433780670166,0.3407149612903595,0.47850000858306885,0.4794999957084656,0.3058593273162842,0.3225170373916626
+edu_fineweb_350b_tokens-seed-1,0,14000,0.4495235048234463,0.3499999940395355,0.3269999921321869,0.3959999978542328,0.4749999940395355,0.2280000001192093,0.37599998712539673,0.6890000104904175,0.7009999752044678,0.3779999911785126,0.40400001406669617,0.5389999747276306,0.5220000147819519,0.3227299153804779,0.33800336718559265,0.4740000069141388,0.4715000092983246,0.306130588054657,0.3196880519390106
+edu_fineweb_350b_tokens-seed-1,0,16000,0.45921100676059723,0.3700000047683716,0.3330000042915344,0.4020000100135803,0.4690000116825104,0.25999999046325684,0.38600000739097595,0.699999988079071,0.7279999852180481,0.38100001215934753,0.4129999876022339,0.5350000262260437,0.5260000228881836,0.32965603470802307,0.35061872005462646,0.5005000233650208,0.4860000014305115,0.31190600991249084,0.33268803358078003
+edu_fineweb_350b_tokens-seed-1,0,18000,0.46205566823482513,0.3709999918937683,0.33899998664855957,0.4009999930858612,0.49799999594688416,0.2280000001192093,0.3779999911785126,0.6890000104904175,0.7160000205039978,0.38199999928474426,0.41499999165534973,0.5339999794960022,0.5210000276565552,0.33332228660583496,0.3538905084133148,0.4970000088214874,0.4934999942779541,0.3169640302658081,0.33594533801078796
+edu_fineweb_350b_tokens-seed-1,0,20000,0.4671029560267925,0.3919999897480011,0.3490000069141388,0.4129999876022339,0.49399998784065247,0.25600001215934753,0.3799999952316284,0.7070000171661377,0.7319999933242798,0.37400001287460327,0.4059999883174896,0.5249999761581421,0.5289999842643738,0.3345783054828644,0.35636845231056213,0.5044999718666077,0.5090000033378601,0.3170454502105713,0.33782368898391724
+edu_fineweb_350b_tokens-seed-1,0,22000,0.463470172137022,0.3889999985694885,0.3310000002384186,0.4169999957084656,0.5,0.25200000405311584,0.37599998712539673,0.6899999976158142,0.7269999980926514,0.3840000033378601,0.4020000100135803,0.5550000071525574,0.5400000214576721,0.32882845401763916,0.3514060974121094,0.4909999966621399,0.49900001287460327,0.31061139702796936,0.33276134729385376
+edu_fineweb_350b_tokens-seed-1,0,24000,0.4662625528872013,0.39800000190734863,0.3409999907016754,0.41200000047683716,0.515999972820282,0.25,0.3700000047683716,0.6949999928474426,0.7329999804496765,0.382999986410141,0.3959999978542328,0.5509999990463257,0.5400000214576721,0.3372650444507599,0.3554573357105255,0.5095000267028809,0.4970000088214874,0.3195478320121765,0.33710044622421265
+edu_fineweb_350b_tokens-seed-1,0,26000,0.4668670482933521,0.3930000066757202,0.32899999618530273,0.421999990940094,0.5180000066757202,0.2619999945163727,0.3840000033378601,0.7049999833106995,0.7239999771118164,0.39100000262260437,0.3930000066757202,0.5550000071525574,0.5360000133514404,0.33861491084098816,0.3594595789909363,0.5105000138282776,0.5095000267028809,0.3203679025173187,0.34143635630607605
+edu_fineweb_350b_tokens-seed-1,0,28000,0.4710822217166424,0.4129999876022339,0.3490000069141388,0.4189999997615814,0.5239999890327454,0.2460000067949295,0.37599998712539673,0.7129999995231628,0.7360000014305115,0.3889999985694885,0.40799999237060547,0.5550000071525574,0.5410000085830688,0.33095699548721313,0.35399219393730164,0.4975000023841858,0.5,0.31196850538253784,0.33465778827667236
+edu_fineweb_350b_tokens-seed-1,0,30000,0.4734017364680767,0.40400001406669617,0.3499999940395355,0.4269999861717224,0.5189999938011169,0.2460000067949295,0.3779999911785126,0.7139999866485596,0.7409999966621399,0.3970000147819519,0.4050000011920929,0.5550000071525574,0.5379999876022339,0.34207814931869507,0.36481064558029175,0.512499988079071,0.5095000267028809,0.3239838480949402,0.3467139005661011
+edu_fineweb_350b_tokens-seed-1,0,32000,0.4748654440045357,0.4050000011920929,0.3499999940395355,0.42500001192092896,0.5329999923706055,0.2540000081062317,0.39800000190734863,0.699999988079071,0.7260000109672546,0.3779999911785126,0.40299999713897705,0.5609999895095825,0.5370000004768372,0.3364258408546448,0.36298680305480957,0.5120000243186951,0.5074999928474426,0.3179066479206085,0.3444235622882843
+edu_fineweb_350b_tokens-seed-1,0,34000,0.4766022339463234,0.39500001072883606,0.3529999852180481,0.4230000078678131,0.5329999923706055,0.24199999868869781,0.3799999952316284,0.6970000267028809,0.7390000224113464,0.38999998569488525,0.4099999964237213,0.5490000247955322,0.5400000214576721,0.3360733985900879,0.35897108912467957,0.503000020980835,0.5184999704360962,0.3182942271232605,0.3393178880214691
+edu_fineweb_350b_tokens-seed-1,0,36000,0.4756137728691101,0.4050000011920929,0.3619999885559082,0.42800000309944153,0.5320000052452087,0.2639999985694885,0.38199999928474426,0.7089999914169312,0.7409999966621399,0.3869999945163727,0.39800000190734863,0.5540000200271606,0.5440000295639038,0.3384101688861847,0.3646673560142517,0.5059999823570251,0.49950000643730164,0.31995898485183716,0.3464101552963257
+edu_fineweb_350b_tokens-seed-1,0,38000,0.4826180450618267,0.4059999883174896,0.375,0.4230000078678131,0.5289999842643738,0.2619999945163727,0.39399999380111694,0.7080000042915344,0.7400000095367432,0.41100001335144043,0.4000000059604645,0.5590000152587891,0.5479999780654907,0.34553104639053345,0.37065890431404114,0.5239999890327454,0.5230000019073486,0.3270617127418518,0.3519443869590759
+edu_fineweb_350b_tokens-seed-1,0,40000,0.4801179841160774,0.40299999713897705,0.35499998927116394,0.4300000071525574,0.5400000214576721,0.2619999945163727,0.39800000190734863,0.7049999833106995,0.746999979019165,0.4020000100135803,0.3970000147819519,0.5649999976158142,0.5540000200271606,0.33888015151023865,0.36192768812179565,0.5145000219345093,0.5074999928474426,0.3198457956314087,0.3424438536167145
+edu_fineweb_350b_tokens-seed-1,0,42000,0.47703278809785843,0.4050000011920929,0.367000013589859,0.4269999861717224,0.5350000262260437,0.2540000081062317,0.3720000088214874,0.703000009059906,0.734000027179718,0.40299999713897705,0.39899998903274536,0.5460000038146973,0.5339999794960022,0.34130677580833435,0.3668607473373413,0.5264999866485596,0.5270000100135803,0.32270070910453796,0.3482622504234314
+edu_fineweb_350b_tokens-seed-1,0,44000,0.4772367440164089,0.4090000092983246,0.3490000069141388,0.43299999833106995,0.5370000004768372,0.25,0.3880000114440918,0.7110000252723694,0.7360000014305115,0.3919999897480011,0.3959999978542328,0.5490000247955322,0.5479999780654907,0.34139013290405273,0.3680916130542755,0.5105000138282776,0.5139999985694885,0.3232519030570984,0.3498939573764801
+edu_fineweb_350b_tokens-seed-1,0,46000,0.4836798347532749,0.4099999964237213,0.36500000953674316,0.4230000078678131,0.5450000166893005,0.25600001215934753,0.38999998569488525,0.7120000123977661,0.746999979019165,0.38999998569488525,0.39899998903274536,0.5550000071525574,0.5490000247955322,0.34736892580986023,0.36823850870132446,0.5299999713897705,0.5254999995231628,0.3293505311012268,0.3489386737346649
+edu_fineweb_350b_tokens-seed-1,0,48000,0.47607188299298286,0.40400001406669617,0.3630000054836273,0.4350000023841858,0.5389999747276306,0.25,0.37599998712539673,0.7170000076293945,0.7409999966621399,0.3919999897480011,0.4000000059604645,0.5540000200271606,0.5429999828338623,0.33596572279930115,0.35683509707450867,0.5120000243186951,0.5095000267028809,0.3168731927871704,0.3370750844478607
+edu_fineweb_350b_tokens-seed-1,0,50000,0.47911832481622696,0.4099999964237213,0.3499999940395355,0.4300000071525574,0.5479999780654907,0.25200000405311584,0.37599998712539673,0.7099999785423279,0.7419999837875366,0.3880000114440918,0.3869999945163727,0.5490000247955322,0.5440000295639038,0.3498813509941101,0.3725147545337677,0.531499981880188,0.531499981880188,0.3322857618331909,0.3544466495513916
+edu_fineweb_350b_tokens-seed-1,0,52000,0.48371143266558647,0.41499999165534973,0.375,0.4350000023841858,0.5540000200271606,0.25600001215934753,0.39399999380111694,0.7179999947547913,0.7459999918937683,0.3880000114440918,0.3959999978542328,0.5600000023841858,0.5360000133514404,0.34429991245269775,0.3713294565677643,0.5199999809265137,0.515999972820282,0.32574552297592163,0.3526914715766907
+edu_fineweb_350b_tokens-seed-1,0,54000,0.4845060631632805,0.41999998688697815,0.367000013589859,0.4410000145435333,0.5460000038146973,0.2460000067949295,0.3700000047683716,0.7049999833106995,0.7459999918937683,0.39899998903274536,0.40400001406669617,0.5569999814033508,0.5569999814033508,0.3471878468990326,0.3677656352519989,0.534500002861023,0.5379999876022339,0.3286001682281494,0.34804850816726685
+edu_fineweb_350b_tokens-seed-1,0,56000,0.48096122220158577,0.40400001406669617,0.3580000102519989,0.4399999976158142,0.5389999747276306,0.27000001072883606,0.40400001406669617,0.7110000252723694,0.7360000014305115,0.38999998569488525,0.39100000262260437,0.5619999766349792,0.5490000247955322,0.354465126991272,0.37144333124160767,0.5195000171661377,0.5174999833106995,0.3372672498226166,0.35318976640701294
+edu_fineweb_350b_tokens-seed-1,0,58000,0.48549820110201836,0.4090000092983246,0.3619999885559082,0.44200000166893005,0.5460000038146973,0.25,0.38999998569488525,0.7279999852180481,0.7480000257492065,0.39100000262260437,0.4020000100135803,0.5619999766349792,0.5490000247955322,0.351541131734848,0.3733488619327545,0.5299999713897705,0.5325000286102295,0.333476722240448,0.35448554158210754
+edu_fineweb_350b_tokens-seed-1,0,60000,0.4874906800687313,0.41100001335144043,0.36399999260902405,0.4350000023841858,0.5519999861717224,0.2639999985694885,0.3919999897480011,0.718999981880188,0.7509999871253967,0.39800000190734863,0.40400001406669617,0.5730000138282776,0.5429999828338623,0.34932613372802734,0.37534230947494507,0.5385000109672546,0.5375000238418579,0.3303368091583252,0.3564254641532898
+edu_fineweb_350b_tokens-seed-1,0,62000,0.48836689814925194,0.4090000092983246,0.36399999260902405,0.4359999895095825,0.5580000281333923,0.27000001072883606,0.4020000100135803,0.7260000109672546,0.7519999742507935,0.40799999237060547,0.39500001072883606,0.5669999718666077,0.550000011920929,0.34964632987976074,0.37406620383262634,0.5389999747276306,0.531000018119812,0.33040371537208557,0.3549351394176483
+edu_fineweb_350b_tokens-seed-1,0,64000,0.48763588443398476,0.4359999895095825,0.3580000102519989,0.44200000166893005,0.5600000023841858,0.24400000274181366,0.37400001287460327,0.7289999723434448,0.7519999742507935,0.39399999380111694,0.4050000011920929,0.5659999847412109,0.5490000247955322,0.3522980511188507,0.37986865639686584,0.5414999723434448,0.5414999723434448,0.3334276080131531,0.3615870773792267
+edu_fineweb_350b_tokens-seed-1,0,66000,0.4877283312380314,0.41499999165534973,0.3619999885559082,0.4390000104904175,0.5479999780654907,0.2619999945163727,0.3799999952316284,0.7229999899864197,0.75,0.4000000059604645,0.4009999930858612,0.5730000138282776,0.5619999766349792,0.3582947850227356,0.3783249855041504,0.5389999747276306,0.5389999747276306,0.3403361439704895,0.3598267436027527
+edu_fineweb_350b_tokens-seed-1,0,68000,0.49183737486600876,0.4269999861717224,0.37599998712539673,0.44999998807907104,0.5640000104904175,0.2680000066757202,0.3919999897480011,0.7239999771118164,0.7429999709129333,0.4000000059604645,0.39500001072883606,0.5720000267028809,0.5529999732971191,0.35574087500572205,0.3818822503089905,0.5419999957084656,0.5485000014305115,0.33680978417396545,0.36319905519485474
+edu_fineweb_350b_tokens-seed-1,0,70000,0.49102676659822464,0.4189999997615814,0.35899999737739563,0.4480000138282776,0.5659999847412109,0.25600001215934753,0.38999998569488525,0.7149999737739563,0.7459999918937683,0.4000000059604645,0.3959999978542328,0.5529999732971191,0.5649999976158142,0.35513052344322205,0.379595547914505,0.5274999737739563,0.5454999804496765,0.337499737739563,0.3607141971588135
+edu_fineweb_350b_tokens-seed-1,0,72000,0.4916461184620857,0.4169999957084656,0.35600000619888306,0.45500001311302185,0.5649999976158142,0.257999986410141,0.38999998569488525,0.7350000143051147,0.75,0.3869999945163727,0.4000000059604645,0.5590000152587891,0.5529999732971191,0.3563148081302643,0.38194817304611206,0.5429999828338623,0.5559999942779541,0.3379554748535156,0.36316898465156555
+edu_fineweb_350b_tokens-seed-1,0,74000,0.49113815650343895,0.41100001335144043,0.36399999260902405,0.45899999141693115,0.5659999847412109,0.2720000147819519,0.3880000114440918,0.7329999804496765,0.7459999918937683,0.3889999985694885,0.40700000524520874,0.5600000023841858,0.5529999732971191,0.356362909078598,0.38203853368759155,0.5339999794960022,0.5414999723434448,0.33809807896614075,0.3636053204536438
+edu_fineweb_350b_tokens-seed-1,0,76000,0.4931333474814892,0.4169999957084656,0.3630000054836273,0.453000009059906,0.5569999814033508,0.27399998903274536,0.40400001406669617,0.7279999852180481,0.746999979019165,0.4009999930858612,0.40400001406669617,0.5529999732971191,0.5479999780654907,0.3585507571697235,0.38335856795310974,0.5429999828338623,0.5575000047683716,0.34024208784103394,0.3645668029785156
+edu_fineweb_350b_tokens-seed-1,0,78000,0.49161795899271965,0.4180000126361847,0.3619999885559082,0.453000009059906,0.5630000233650208,0.27799999713897705,0.38999998569488525,0.7329999804496765,0.7419999837875366,0.40400001406669617,0.40400001406669617,0.5580000281333923,0.5580000281333923,0.358989417552948,0.3820967674255371,0.5460000038146973,0.5504999756813049,0.34032127261161804,0.36344367265701294
+edu_fineweb_350b_tokens-seed-1,0,80000,0.4911441020667553,0.41600000858306885,0.35899999737739563,0.44699999690055847,0.5740000009536743,0.2619999945163727,0.39399999380111694,0.7300000190734863,0.7540000081062317,0.39800000190734863,0.39800000190734863,0.5490000247955322,0.5419999957084656,0.35973435640335083,0.3821263015270233,0.5370000004768372,0.5444999933242798,0.3422233760356903,0.3636528253555298
+edu_fineweb_350b_tokens-seed-1,0,82000,0.4927275590598583,0.42399999499320984,0.36500000953674316,0.4410000145435333,0.5580000281333923,0.2919999957084656,0.3959999978542328,0.7250000238418579,0.7390000224113464,0.39100000262260437,0.40700000524520874,0.5720000267028809,0.5590000152587891,0.36026227474212646,0.38345789909362793,0.5600000023841858,0.5529999732971191,0.341264009475708,0.3648204207420349
+edu_fineweb_350b_tokens-seed-1,0,84000,0.49356314539909363,0.4350000023841858,0.367000013589859,0.45399999618530273,0.5659999847412109,0.2800000011920929,0.39800000190734863,0.7260000109672546,0.7369999885559082,0.39399999380111694,0.40700000524520874,0.5450000166893005,0.5529999732971191,0.3620130121707916,0.38375842571258545,0.5640000104904175,0.5554999709129333,0.3433130979537964,0.365005224943161
+edu_fineweb_350b_tokens-seed-1,0,86000,0.4939306415617466,0.42100000381469727,0.367000013589859,0.45100000500679016,0.5709999799728394,0.2840000092983246,0.3880000114440918,0.7269999980926514,0.7519999742507935,0.39399999380111694,0.4059999883174896,0.5820000171661377,0.5669999718666077,0.3596394956111908,0.3779442608356476,0.5419999957084656,0.5419999957084656,0.34093979001045227,0.3584451973438263
+edu_fineweb_350b_tokens-seed-1,0,88000,0.5010630041360855,0.42500001192092896,0.3709999918937683,0.45500001311302185,0.5789999961853027,0.2680000066757202,0.39800000190734863,0.7310000061988831,0.7540000081062317,0.41100001335144043,0.41200000047683716,0.5839999914169312,0.5649999976158142,0.3666640520095825,0.3865112364292145,0.5575000047683716,0.5625,0.34814321994781494,0.36700403690338135
+edu_fineweb_350b_tokens-seed-1,0,90000,0.49995486810803413,0.41100001335144043,0.3630000054836273,0.4560000002384186,0.574999988079071,0.27399998903274536,0.40400001406669617,0.7300000190734863,0.7570000290870667,0.4059999883174896,0.41200000047683716,0.5709999799728394,0.5600000023841858,0.3633595407009125,0.38647565245628357,0.5565000176429749,0.5615000128746033,0.34486615657806396,0.36713889241218567
+edu_fineweb_350b_tokens-seed-1,0,92000,0.49723950773477554,0.42899999022483826,0.3709999918937683,0.45399999618530273,0.5649999976158142,0.2800000011920929,0.39800000190734863,0.7329999804496765,0.7519999742507935,0.3970000147819519,0.4180000126361847,0.578000009059906,0.5619999766349792,0.3627259135246277,0.3835802674293518,0.546500027179718,0.5475000143051147,0.3440909683704376,0.36441609263420105
+edu_fineweb_350b_tokens-seed-1,0,94000,0.4972243830561638,0.42500001192092896,0.3610000014305115,0.4519999921321869,0.5709999799728394,0.2840000092983246,0.40400001406669617,0.7279999852180481,0.7459999918937683,0.4050000011920929,0.41499999165534973,0.5690000057220459,0.5600000023841858,0.36130890250205994,0.3823433518409729,0.5559999942779541,0.5580000281333923,0.34228208661079407,0.36279505491256714
+edu_fineweb_350b_tokens-seed-1,0,96000,0.4945024959743023,0.41100001335144043,0.38100001215934753,0.45500001311302185,0.5730000138282776,0.2759999930858612,0.38600000739097595,0.7329999804496765,0.75,0.39899998903274536,0.4000000059604645,0.5720000267028809,0.5509999990463257,0.3607921898365021,0.382955938577652,0.5414999723434448,0.5509999990463257,0.34249985218048096,0.3640199303627014
+edu_fineweb_350b_tokens-seed-1,0,98000,0.4953378774225712,0.40700000524520874,0.367000013589859,0.46299999952316284,0.5740000009536743,0.27799999713897705,0.3919999897480011,0.7269999980926514,0.7559999823570251,0.39399999380111694,0.40700000524520874,0.5759999752044678,0.5540000200271606,0.3606520891189575,0.3814857304096222,0.5460000038146973,0.5504999756813049,0.34219974279403687,0.36220303177833557
+edu_fineweb_350b_tokens-seed-1,0,100000,0.499586995691061,0.41200000047683716,0.3659999966621399,0.4620000123977661,0.5830000042915344,0.26600000262260437,0.4000000059604645,0.7250000238418579,0.7590000033378601,0.4009999930858612,0.4099999964237213,0.5720000267028809,0.5720000267028809,0.35491371154785156,0.3793718218803406,0.5419999957084656,0.5475000143051147,0.3359191417694092,0.35919591784477234
+edu_fineweb_350b_tokens-seed-1,0,102000,0.4977268613874912,0.41499999165534973,0.38199999928474426,0.4580000042915344,0.574999988079071,0.2720000147819519,0.414000004529953,0.7229999899864197,0.7509999871253967,0.3970000147819519,0.4129999876022339,0.5619999766349792,0.5509999990463257,0.3581334054470062,0.3803453743457794,0.5354999899864197,0.5350000262260437,0.3400118052959442,0.3608148992061615
+edu_fineweb_350b_tokens-seed-1,0,104000,0.4989877715706825,0.41100001335144043,0.36000001430511475,0.46299999952316284,0.5709999799728394,0.27399998903274536,0.39399999380111694,0.7329999804496765,0.7570000290870667,0.414000004529953,0.41499999165534973,0.5709999799728394,0.5590000152587891,0.3643620014190674,0.38796037435531616,0.559499979019165,0.5669999718666077,0.34558823704719543,0.36890217661857605
+edu_fineweb_350b_tokens-seed-1,0,106000,0.4987182281911373,0.41499999165534973,0.3700000047683716,0.4659999907016754,0.578000009059906,0.2840000092983246,0.4059999883174896,0.7260000109672546,0.7480000257492065,0.40700000524520874,0.4090000092983246,0.5680000185966492,0.5550000071525574,0.3635769784450531,0.38422322273254395,0.5450000166893005,0.5590000152587891,0.3452018201351166,0.36474576592445374
+edu_fineweb_350b_tokens-seed-1,0,108000,0.5014015696942806,0.4320000112056732,0.36899998784065247,0.4620000123977661,0.5860000252723694,0.2840000092983246,0.40400001406669617,0.7350000143051147,0.75,0.40299999713897705,0.41100001335144043,0.5630000233650208,0.5699999928474426,0.36332157254219055,0.3844863772392273,0.5550000071525574,0.5565000176429749,0.34433162212371826,0.36471250653266907
+edu_fineweb_350b_tokens-seed-1,0,110000,0.5043375752866268,0.4230000078678131,0.37700000405311584,0.4620000123977661,0.5730000138282776,0.2879999876022339,0.41200000047683716,0.7419999837875366,0.7749999761581421,0.39899998903274536,0.414000004529953,0.5720000267028809,0.5590000152587891,0.36554232239723206,0.3870066702365875,0.5554999709129333,0.5575000047683716,0.3467237055301666,0.3672005832195282
+edu_fineweb_350b_tokens-seed-1,0,112000,0.5046022236347198,0.4020000100135803,0.36800000071525574,0.4650000035762787,0.5839999914169312,0.2840000092983246,0.4300000071525574,0.7310000061988831,0.7540000081062317,0.40400001406669617,0.40700000524520874,0.5720000267028809,0.5569999814033508,0.367435097694397,0.39113253355026245,0.5565000176429749,0.5649999976158142,0.34933826327323914,0.37181779742240906
+edu_fineweb_350b_tokens-seed-1,0,114000,0.5044217295944691,0.4230000078678131,0.3709999918937683,0.4659999907016754,0.5830000042915344,0.2939999997615814,0.4180000126361847,0.7289999723434448,0.7549999952316284,0.4020000100135803,0.40700000524520874,0.5709999799728394,0.5640000104904175,0.36639603972435,0.38989707827568054,0.5580000281333923,0.5669999718666077,0.34762707352638245,0.37037384510040283
+edu_fineweb_350b_tokens-seed-1,0,116000,0.5037211365997791,0.41600000858306885,0.3720000088214874,0.46399998664855957,0.5899999737739563,0.3059999942779541,0.41200000047683716,0.7300000190734863,0.7540000081062317,0.4020000100135803,0.41200000047683716,0.5659999847412109,0.5600000023841858,0.3651641011238098,0.3900282680988312,0.5569999814033508,0.5590000152587891,0.34627485275268555,0.37076908349990845
+edu_fineweb_350b_tokens-seed-1,0,118000,0.5019807890057564,0.41200000047683716,0.3799999952316284,0.46399998664855957,0.5889999866485596,0.2879999876022339,0.4059999883174896,0.7419999837875366,0.7630000114440918,0.4009999930858612,0.39800000190734863,0.5600000023841858,0.5609999895095825,0.36456480622291565,0.3854806423187256,0.5534999966621399,0.5529999732971191,0.34601250290870667,0.3658463656902313
+edu_fineweb_350b_tokens-seed-1,0,120000,0.5008365102112293,0.4259999990463257,0.367000013589859,0.47200000286102295,0.5799999833106995,0.29600000381469727,0.41600000858306885,0.734000027179718,0.7580000162124634,0.3970000147819519,0.40400001406669617,0.5559999942779541,0.5550000071525574,0.3660300672054291,0.3887145519256592,0.5529999732971191,0.5569999814033508,0.34745535254478455,0.3696920573711395
+edu_fineweb_350b_tokens-seed-1,0,122000,0.5028450302779675,0.41999998688697815,0.3700000047683716,0.4659999907016754,0.5929999947547913,0.2919999957084656,0.4020000100135803,0.7400000095367432,0.7590000033378601,0.4000000059604645,0.40400001406669617,0.5659999847412109,0.574999988079071,0.365267276763916,0.3863743841648102,0.5540000200271606,0.5529999732971191,0.3464977741241455,0.36676025390625
+edu_fineweb_350b_tokens-seed-1,0,124000,0.5072730705142021,0.41499999165534973,0.3720000088214874,0.47099998593330383,0.5860000252723694,0.2919999957084656,0.42800000309944153,0.734000027179718,0.7620000243186951,0.3930000066757202,0.40700000524520874,0.5669999718666077,0.5649999976158142,0.365468293428421,0.3888309895992279,0.5544999837875366,0.5695000290870667,0.3469199538230896,0.36868447065353394
+edu_fineweb_350b_tokens-seed-1,0,126000,0.501251045614481,0.4180000126361847,0.3659999966621399,0.47099998593330383,0.5839999914169312,0.2879999876022339,0.41999998688697815,0.7369999885559082,0.7630000114440918,0.39500001072883606,0.4059999883174896,0.5600000023841858,0.5580000281333923,0.3610328137874603,0.3824765980243683,0.5485000014305115,0.5504999756813049,0.3421251177787781,0.3625083863735199
+edu_fineweb_350b_tokens-seed-1,0,128000,0.5086825042963028,0.42800000309944153,0.3779999911785126,0.46399998664855957,0.5860000252723694,0.28200000524520874,0.41999998688697815,0.7369999885559082,0.7580000162124634,0.3869999945163727,0.41499999165534973,0.5730000138282776,0.5680000185966492,0.3646826446056366,0.3922956585884094,0.5584999918937683,0.5720000267028809,0.3459012806415558,0.3724599778652191
+edu_fineweb_350b_tokens-seed-1,0,130000,0.5078329555690289,0.4339999854564667,0.37400001287460327,0.46299999952316284,0.5849999785423279,0.2919999957084656,0.41999998688697815,0.7350000143051147,0.7570000290870667,0.3889999985694885,0.41600000858306885,0.5669999718666077,0.5720000267028809,0.3667779266834259,0.39438962936401367,0.559499979019165,0.5634999871253967,0.34809762239456177,0.37516361474990845
+edu_fineweb_350b_tokens-seed-1,0,132000,0.5061904042959213,0.4230000078678131,0.3659999966621399,0.4659999907016754,0.5879999995231628,0.28999999165534973,0.41999998688697815,0.746999979019165,0.7630000114440918,0.4050000011920929,0.4090000092983246,0.5820000171661377,0.5659999847412109,0.3683132827281952,0.39066654443740845,0.5619999766349792,0.5665000081062317,0.34918180108070374,0.3710232377052307
+edu_fineweb_350b_tokens-seed-1,0,134000,0.5059116296470165,0.4189999997615814,0.3709999918937683,0.47600001096725464,0.5870000123977661,0.2939999997615814,0.41999998688697815,0.7409999966621399,0.7570000290870667,0.4000000059604645,0.4099999964237213,0.5720000267028809,0.5669999718666077,0.3706933557987213,0.3925185203552246,0.5615000128746033,0.5619999766349792,0.3521064519882202,0.37329307198524475
+edu_fineweb_350b_tokens-seed-1,0,138000,0.5042317919433117,0.4230000078678131,0.367000013589859,0.4740000069141388,0.5860000252723694,0.2879999876022339,0.41600000858306885,0.7390000224113464,0.765999972820282,0.3959999978542328,0.3970000147819519,0.5740000009536743,0.5609999895095825,0.370014488697052,0.39456456899642944,0.5630000233650208,0.5649999976158142,0.35142001509666443,0.3758543133735657
+edu_fineweb_350b_tokens-seed-1,0,140000,0.5045665018260479,0.41999998688697815,0.3799999952316284,0.4690000116825104,0.5820000171661377,0.2720000147819519,0.4099999964237213,0.7369999885559082,0.7670000195503235,0.3970000147819519,0.4000000059604645,0.5609999895095825,0.5590000152587891,0.37111103534698486,0.3926049470901489,0.5590000152587891,0.5649999976158142,0.35347750782966614,0.3735319674015045
+edu_fineweb_350b_tokens-seed-1,0,142000,0.5085432901978493,0.421999990940094,0.3709999918937683,0.47200000286102295,0.5950000286102295,0.27000001072883606,0.414000004529953,0.7329999804496765,0.7630000114440918,0.3930000066757202,0.40400001406669617,0.5720000267028809,0.5680000185966492,0.369999498128891,0.39443445205688477,0.565500020980835,0.578499972820282,0.3518766164779663,0.37484627962112427
+edu_fineweb_350b_tokens-seed-1,0,144000,0.5049711987376213,0.43299999833106995,0.36899998784065247,0.46399998664855957,0.5989999771118164,0.2720000147819519,0.40799999237060547,0.7329999804496765,0.7590000033378601,0.3880000114440918,0.3959999978542328,0.5659999847412109,0.5690000057220459,0.36937984824180603,0.3920287489891052,0.5665000081062317,0.5669999718666077,0.3512401580810547,0.37276965379714966
+edu_fineweb_350b_tokens-seed-1,0,148000,0.5123470723628998,0.4269999861717224,0.3790000081062317,0.47099998593330383,0.5950000286102295,0.2800000011920929,0.42399999499320984,0.7310000061988831,0.7649999856948853,0.39899998903274536,0.41100001335144043,0.5770000219345093,0.5770000219345093,0.36998581886291504,0.3943348228931427,0.5674999952316284,0.5734999775886536,0.3513873219490051,0.3742765486240387
+edu_fineweb_350b_tokens-seed-1,0,150000,0.509139034897089,0.41100001335144043,0.3779999911785126,0.4740000069141388,0.593999981880188,0.2720000147819519,0.41200000047683716,0.7319999933242798,0.7649999856948853,0.4009999930858612,0.40400001406669617,0.5680000185966492,0.5649999976158142,0.3709569573402405,0.3972984850406647,0.5740000009536743,0.5770000219345093,0.35274040699005127,0.3781122863292694
+edu_fineweb_350b_tokens-seed-1,0,152000,0.5079653523862362,0.4169999957084656,0.36800000071525574,0.4729999899864197,0.5960000157356262,0.2800000011920929,0.40799999237060547,0.7390000224113464,0.7630000114440918,0.39800000190734863,0.40400001406669617,0.5720000267028809,0.574999988079071,0.3738082945346832,0.3963184654712677,0.5715000033378601,0.5724999904632568,0.3557112216949463,0.37722280621528625
+edu_fineweb_350b_tokens-seed-1,0,154000,0.5107964277267456,0.4230000078678131,0.3700000047683716,0.4749999940395355,0.5960000157356262,0.2879999876022339,0.41999998688697815,0.7350000143051147,0.7639999985694885,0.4000000059604645,0.41200000047683716,0.5730000138282776,0.5709999799728394,0.37119078636169434,0.39557957649230957,0.5720000267028809,0.5774999856948853,0.3524456322193146,0.3758714497089386
+edu_fineweb_350b_tokens-seed-1,0,156000,0.5088927485048771,0.41999998688697815,0.3709999918937683,0.46700000762939453,0.5910000205039978,0.2919999957084656,0.414000004529953,0.7369999885559082,0.765999972820282,0.39399999380111694,0.4059999883174896,0.5799999833106995,0.5669999718666077,0.3713153600692749,0.3947707414627075,0.5745000243186951,0.5809999704360962,0.35257014632225037,0.37514206767082214
+edu_fineweb_350b_tokens-seed-1,0,158000,0.5088703669607639,0.41999998688697815,0.375,0.4699999988079071,0.5979999899864197,0.28200000524520874,0.40400001406669617,0.7450000047683716,0.7680000066757202,0.39500001072883606,0.4009999930858612,0.574999988079071,0.574999988079071,0.36792638897895813,0.3940982520580292,0.5680000185966492,0.5755000114440918,0.3489862382411957,0.37446293234825134
+edu_fineweb_350b_tokens-seed-1,0,160000,0.5071291252970695,0.4300000071525574,0.35899999737739563,0.4729999899864197,0.5929999947547913,0.28200000524520874,0.4180000126361847,0.7440000176429749,0.7630000114440918,0.3919999897480011,0.4020000100135803,0.5759999752044678,0.574999988079071,0.36913686990737915,0.3938981890678406,0.5669999718666077,0.5724999904632568,0.3502262532711029,0.3745329976081848
+edu_fineweb_350b_tokens-seed-1,0,162000,0.5091184116899967,0.41600000858306885,0.367000013589859,0.4740000069141388,0.5920000076293945,0.2879999876022339,0.40799999237060547,0.746999979019165,0.7689999938011169,0.38999998569488525,0.4090000092983246,0.5720000267028809,0.5770000219345093,0.36741992831230164,0.39286142587661743,0.5720000267028809,0.578000009059906,0.3482683300971985,0.37294724583625793
+edu_fineweb_350b_tokens-seed-1,0,164000,0.5078432261943817,0.41600000858306885,0.36500000953674316,0.46700000762939453,0.5910000205039978,0.2759999930858612,0.40799999237060547,0.7369999885559082,0.7689999938011169,0.39500001072883606,0.4059999883174896,0.5759999752044678,0.5799999833106995,0.36831894516944885,0.3920001685619354,0.5634999871253967,0.5715000033378601,0.3499426543712616,0.37224581837654114
+edu_fineweb_350b_tokens-seed-1,0,166000,0.5083079226315022,0.41499999165534973,0.36399999260902405,0.47200000286102295,0.5929999947547913,0.28200000524520874,0.414000004529953,0.7400000095367432,0.7680000066757202,0.4009999930858612,0.40799999237060547,0.574999988079071,0.5699999928474426,0.37059345841407776,0.3931756019592285,0.5640000104904175,0.5759999752044678,0.3522030711174011,0.3734634220600128
+edu_fineweb_350b_tokens-seed-1,0,167000,0.509494174271822,0.42899999022483826,0.3619999885559082,0.47200000286102295,0.597000002861023,0.28999999165534973,0.4180000126361847,0.7379999756813049,0.7689999938011169,0.39500001072883606,0.40400001406669617,0.5820000171661377,0.578000009059906,0.3696656823158264,0.3941360414028168,0.5669999718666077,0.5734999775886536,0.3506713807582855,0.3744533956050873

notebooks/minhash_params.ipynb

@@ -0,0 +1,174 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'prob': {'file': 'prob.json'}}"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import json\n",
+    "import numpy as np\n",
+    "import plotly.graph_objects as go\n",
+    "RED_FULL=\"rgba(255, 0, 0, 1)\"\n",
+    "\n",
+    "# Define the function 1 - (1 - x^8)^14\n",
+    "def func1(x):\n",
+    "    return 1 - np.power(1 - np.power(x, 8), 14)\n",
+    "\n",
+    "# Define the function 1 - (1 - x^20)^450\n",
+    "def func2(x):\n",
+    "    return 1 - np.power(1 - np.power(x, 20), 450)\n",
+    "\n",
+    "# Generate x values from 0 to 1\n",
+    "x = np.linspace(0, 1, 1000)\n",
+    "\n",
+    "# Calculate y values for each function\n",
+    "y1 = func1(x)\n",
+    "y2 = func2(x)\n",
+    "\n",
+    "# Create traces\n",
+    "trace1 = go.Scatter(x=x, y=y1, mode='lines', name='FineWeb: 1-(1-s^8)^14')\n",
+    "trace2 = go.Scatter(x=x, y=y2, mode='lines', name='RefinedWeb: 1-(1-s^20)^450')\n",
+    "vertical_line = go.Scatter(x=[0.75, 0.75], y=[0, 1], mode='lines', line=dict(color='red', dash='dash'), name='Threshold')\n",
+    "\n",
+    "# Define layout\n",
+    "layout = {\n",
+    "    'title': {\n",
+    "        'text': 'MinHash parameters',\n",
+    "    },\n",
+    "    'xaxis': {\n",
+    "        'title': {\n",
+    "            'text': 'Document similarity (s)',\n",
+    "        },\n",
+    "    },\n",
+    "    'yaxis': {\n",
+    "        'title': {\n",
+    "            'text': 'Matched as dups probability',\n",
+    "        },\n",
+    "    },\n",
+    "}\n",
+    "\n",
+    "\n",
+    "def normalize_run_name(run_name):\n",
+    "    return run_name.replace(\"/\", \"_\")\n",
+    "\n",
+    "\n",
+    "def save_for_plot(dir_name, df, views, xlabel=\"Dataset\", ylabel=\"Matched as dups probability\", plot_name=\"plot name\", custom_layout={}, ranges={}, x_column=None, default_metric=None):\n",
+    "    import os\n",
+    "    files = {}\n",
+    "    os.makedirs(f\"data/plots/{dir_name}\", exist_ok=True)\n",
+    "    for view in views:\n",
+    "        data = {}\n",
+    "        for run_name in df[\"runname\"].unique():\n",
+    "            run_name_only=df[df[\"runname\"]==run_name]\n",
+    "            data[run_name] = {\n",
+    "                \"x\": run_name_only[x_column].tolist() if x_column else [run_name],\n",
+    "                \"y\": run_name_only[view].tolist(),\n",
+    "                \"label\": run_name,\n",
+    "            }\n",
+    "        file_name = f\"{normalize_run_name(view)}.json\"\n",
+    "        files[view] = {\"file\": f\"{file_name}\"}\n",
+    "        with open(f\"data/plots/{dir_name}/{file_name}\", \"w\") as f:\n",
+    "            json.dump({\n",
+    "                \"data\": data,\n",
+    "                \"layout\": {\n",
+    "                    \"title\": {\n",
+    "                        \"text\": plot_name,\n",
+    "                    },\n",
+    "                    \"xaxis\": {\n",
+    "                        \"title\": {\n",
+    "                            \"text\": xlabel,\n",
+    "                        },\n",
+    "                    },\n",
+    "                    \"yaxis\": {\n",
+    "                        # \"range\": ranges.get(view, None),\n",
+    "                        \"title\": {\n",
+    "                            \"text\": ylabel,\n",
+    "                        },\n",
+    "                    },\n",
+    "                    \"shapes\": [\n",
+    "                        {\n",
+    "                            \"type\": \"line\",\n",
+    "                            \"x0\": 0.75,\n",
+    "                            \"y0\": 0.0,\n",
+    "                            \"x1\": 0.75,\n",
+    "                            \"y1\": 1.2,\n",
+    "                            \"xref\": \"x\",\n",
+    "                            \"yref\": \"y\",\n",
+    "                            \"line\": {\n",
+    "                                \"color\": RED_FULL,\n",
+    "                                \"width\": 1,\n",
+    "                                \"dash\": \"dashdot\"\n",
+    "                            },\n",
+    "                            \"showarrow\": False\n",
+    "                        }\n",
+    "        ],\n",
+    "                    **custom_layout,\n",
+    "                },\n",
+    "            }, f)\n",
+    "    with open(f\"data/plots/{dir_name}/index.json\", \"w\") as f:\n",
+    "        json.dump({\n",
+    "            \"files\": files,\n",
+    "            \"settings\": {\n",
+    "                \"defaultMetric\": default_metric,\n",
+    "                \"slider\": None,\n",
+    "                \"autoSetXRange\": False,\n",
+    "            }\n",
+    "        }, f)\n",
+    "    return files\n",
+    "\n",
+    "import pandas as pd\n",
+    "df = pd.DataFrame({\n",
+    "    \"runname\": [\"FineWeb: 1-(1-s^8)^14\"]*len(x) + [\"RefinedWeb: 1-(1-s^20)^450\"]*len(x),\n",
+    "    \"similarity\": x.tolist()+x.tolist(),\n",
+    "    \"prob\": y1.tolist()+y2.tolist(),\n",
+    "    \"view\": [\"normal\"]*2*len(x)\n",
+    "})\n",
+    "\n",
+    "custom_layout = {\n",
+    "    \"legend\": {\n",
+    "        \"orientation\": \"v\",\n",
+    "        \"xanchor\": \"left\",\n",
+    "        \"yanchor\": \"top\",\n",
+    "        \"x\": 0,\n",
+    "        \"y\": 1,\n",
+    "    },\n",
+    "}\n",
+    "\n",
+    "save_for_plot(\"minhash_params\", df, [\"prob\"], xlabel=\"Document similarity (s)\", plot_name=\"MinHash parameters\", custom_layout=custom_layout, ranges={}, x_column=\"similarity\", default_metric=\"prob\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "datatrove",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

notebooks/modify_jsons.ipynb

@@ -0,0 +1,116 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import json\n",
+    "import orjson\n",
+    "\n",
+    "\n",
+    "def normalize_file_name(file_name):\n",
+    "    return file_name.replace('/', '_')\n",
+    "\n",
+    "\n",
+    "def keep_key(key):\n",
+    "    if key.endswith(\"acc\"):\n",
+    "        return False\n",
+    "    \n",
+    "    if \"sciq\" in key:\n",
+    "        return False\n",
+    "\n",
+    "    if \"siqa\" in key:\n",
+    "        return False\n",
+    "\n",
+    "    return True\n",
+    "\n",
+    "\n",
+    "def get_slider_max(data):\n",
+    "    metrics = data[list(data.keys())[0]]\n",
+    "    metric_data = metrics[list(metrics.keys())[0]]\n",
+    "    samples = len(metric_data[\"x\"])\n",
+    "    if samples < 20:\n",
+    "        return 10\n",
+    "    return 30\n",
+    "\n",
+    "\n",
+    "def create_index(data, traces, layout, default_window_size, default_metric):\n",
+    "    print(default_metric if default_metric else \"None\")\n",
+    "    files_data = {}\n",
+    "    index_files = {}\n",
+    "    for task_id, task_data in (data.items() if data else traces.items()):\n",
+    "        data_name = \"data\" if data else \"traces\"\n",
+    "        files_data[task_id] = {\n",
+    "            data_name: task_data,\n",
+    "            \"layout\": layout\n",
+    "        }\n",
+    "        index_files[task_id] = {\n",
+    "            \"file\": f\"{normalize_file_name(task_id)}.json\"\n",
+    "        }\n",
+    "    settings = {\n",
+    "        \"slider\": {\n",
+    "            \"min\": 0,\n",
+    "            \"max\": get_slider_max(data),\n",
+    "            \"default\": default_window_size,\n",
+    "        },\n",
+    "        \"defaultMetric\": default_metric\n",
+    "    } if data else {\"slider\": None}\n",
+    "    \n",
+    "    return files_data, index_files, settings\n",
+    "    \n",
+    "    \n",
+    "\n",
+    "new_data = {}\n",
+    "\n",
+    "for file_name in os.listdir('./data/plots'):\n",
+    "    if not file_name.endswith('.json'):\n",
+    "        continue\n",
+    "    with open(f'./data/plots/{file_name}', 'r') as file:\n",
+    "        old_data = orjson.loads(file.read())\n",
+    "    data = {key: value for key, value in old_data[\"data\"].items() if keep_key(key)} if \"data\" in old_data else {}\n",
+    "    traces = {key: value for key, value in old_data[\"traces\"].items()} if \"traces\" in old_data else {}\n",
+    "    default_window_size = old_data[\"defaultWindowSize\"] if \"defaultWindowSize\" in old_data else None\n",
+    "    default_metric = old_data[\"defaultMetric\"] if \"defaultMetric\" in old_data else None\n",
+    "    files_data, index_files, settings = create_index(data, traces, old_data[\"layout\"], default_window_size, default_metric)\n",
+    "    # mkdir\n",
+    "    dir_name = file_name.split('.')[0]\n",
+    "    os.makedirs(f'./data/plots/{dir_name}', exist_ok=True)\n",
+    "    with open(f'./data/plots/{dir_name}/index.json', 'wb') as file:\n",
+    "        file.write(orjson.dumps({\n",
+    "            \"files\": index_files,\n",
+    "            \"settings\": settings,\n",
+    "        }))\n",
+    "    \n",
+    "    for metric_name, data in files_data.items():\n",
+    "        with open(f'./data/plots/{dir_name}/{normalize_file_name(metric_name)}.json', 'wb') as file:\n",
+    "            file.write(orjson.dumps(data))\n",
+    "\n",
+    "\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "datatrove3.10",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

notebooks/plot_all-filtering-steps.ipynb

@@ -0,0 +1,580 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "138889b92720ce2e",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-14T09:02:09.162993Z",
+     "start_time": "2024-05-14T09:02:09.134625Z"
+    },
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>runname</th>\n",
+       "      <th>seed</th>\n",
+       "      <th>steps</th>\n",
+       "      <th>agg_score</th>\n",
+       "      <th>commonsense_qa/acc</th>\n",
+       "      <th>commonsense_qa/acc_norm</th>\n",
+       "      <th>hellaswag/acc</th>\n",
+       "      <th>hellaswag/acc_norm</th>\n",
+       "      <th>openbookqa/acc</th>\n",
+       "      <th>openbookqa/acc_norm</th>\n",
+       "      <th>...</th>\n",
+       "      <th>siqa/acc</th>\n",
+       "      <th>siqa/acc_norm</th>\n",
+       "      <th>winogrande/acc</th>\n",
+       "      <th>winogrande/acc_norm</th>\n",
+       "      <th>sciq/acc</th>\n",
+       "      <th>sciq/acc_norm</th>\n",
+       "      <th>arc/acc</th>\n",
+       "      <th>arc/acc_norm</th>\n",
+       "      <th>mmlu/acc</th>\n",
+       "      <th>mmlu/acc_norm</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>big-run-sampled-fineweb-c4-filters</td>\n",
+       "      <td>6</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.330893</td>\n",
+       "      <td>0.186</td>\n",
+       "      <td>0.233</td>\n",
+       "      <td>0.272</td>\n",
+       "      <td>0.258</td>\n",
+       "      <td>0.166</td>\n",
+       "      <td>0.286</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.367</td>\n",
+       "      <td>0.362</td>\n",
+       "      <td>0.516</td>\n",
+       "      <td>0.497</td>\n",
+       "      <td>0.208</td>\n",
+       "      <td>0.202</td>\n",
+       "      <td>0.2195</td>\n",
+       "      <td>0.2510</td>\n",
+       "      <td>0.230294</td>\n",
+       "      <td>0.250147</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>big-run-sampled-fineweb-c4-filters</td>\n",
+       "      <td>6</td>\n",
+       "      <td>1000</td>\n",
+       "      <td>0.359303</td>\n",
+       "      <td>0.250</td>\n",
+       "      <td>0.263</td>\n",
+       "      <td>0.293</td>\n",
+       "      <td>0.285</td>\n",
+       "      <td>0.140</td>\n",
+       "      <td>0.276</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.376</td>\n",
+       "      <td>0.401</td>\n",
+       "      <td>0.497</td>\n",
+       "      <td>0.479</td>\n",
+       "      <td>0.594</td>\n",
+       "      <td>0.524</td>\n",
+       "      <td>0.2740</td>\n",
+       "      <td>0.2985</td>\n",
+       "      <td>0.241617</td>\n",
+       "      <td>0.251920</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>big-run-sampled-fineweb-c4-filters</td>\n",
+       "      <td>6</td>\n",
+       "      <td>2000</td>\n",
+       "      <td>0.375393</td>\n",
+       "      <td>0.268</td>\n",
+       "      <td>0.277</td>\n",
+       "      <td>0.319</td>\n",
+       "      <td>0.324</td>\n",
+       "      <td>0.150</td>\n",
+       "      <td>0.274</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.372</td>\n",
+       "      <td>0.411</td>\n",
+       "      <td>0.507</td>\n",
+       "      <td>0.484</td>\n",
+       "      <td>0.688</td>\n",
+       "      <td>0.606</td>\n",
+       "      <td>0.3015</td>\n",
+       "      <td>0.3270</td>\n",
+       "      <td>0.246577</td>\n",
+       "      <td>0.259146</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>big-run-sampled-fineweb-c4-filters</td>\n",
+       "      <td>6</td>\n",
+       "      <td>3000</td>\n",
+       "      <td>0.389655</td>\n",
+       "      <td>0.303</td>\n",
+       "      <td>0.305</td>\n",
+       "      <td>0.324</td>\n",
+       "      <td>0.358</td>\n",
+       "      <td>0.152</td>\n",
+       "      <td>0.280</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.383</td>\n",
+       "      <td>0.389</td>\n",
+       "      <td>0.520</td>\n",
+       "      <td>0.506</td>\n",
+       "      <td>0.741</td>\n",
+       "      <td>0.647</td>\n",
+       "      <td>0.3395</td>\n",
+       "      <td>0.3405</td>\n",
+       "      <td>0.255001</td>\n",
+       "      <td>0.268740</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>big-run-sampled-fineweb-c4-filters</td>\n",
+       "      <td>6</td>\n",
+       "      <td>4000</td>\n",
+       "      <td>0.401195</td>\n",
+       "      <td>0.309</td>\n",
+       "      <td>0.310</td>\n",
+       "      <td>0.353</td>\n",
+       "      <td>0.393</td>\n",
+       "      <td>0.138</td>\n",
+       "      <td>0.288</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.378</td>\n",
+       "      <td>0.402</td>\n",
+       "      <td>0.534</td>\n",
+       "      <td>0.511</td>\n",
+       "      <td>0.766</td>\n",
+       "      <td>0.652</td>\n",
+       "      <td>0.3395</td>\n",
+       "      <td>0.3495</td>\n",
+       "      <td>0.256203</td>\n",
+       "      <td>0.269056</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>667</th>\n",
+       "      <td>big-run-sampled_full_filtered_no_dedup</td>\n",
+       "      <td>6</td>\n",
+       "      <td>163000</td>\n",
+       "      <td>0.466255</td>\n",
+       "      <td>0.426</td>\n",
+       "      <td>0.372</td>\n",
+       "      <td>0.469</td>\n",
+       "      <td>0.555</td>\n",
+       "      <td>0.242</td>\n",
+       "      <td>0.354</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.389</td>\n",
+       "      <td>0.394</td>\n",
+       "      <td>0.563</td>\n",
+       "      <td>0.544</td>\n",
+       "      <td>0.869</td>\n",
+       "      <td>0.808</td>\n",
+       "      <td>0.4460</td>\n",
+       "      <td>0.4435</td>\n",
+       "      <td>0.297125</td>\n",
+       "      <td>0.317543</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>668</th>\n",
+       "      <td>big-run-sampled_full_filtered_no_dedup</td>\n",
+       "      <td>6</td>\n",
+       "      <td>164000</td>\n",
+       "      <td>0.469743</td>\n",
+       "      <td>0.431</td>\n",
+       "      <td>0.376</td>\n",
+       "      <td>0.467</td>\n",
+       "      <td>0.556</td>\n",
+       "      <td>0.232</td>\n",
+       "      <td>0.356</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.391</td>\n",
+       "      <td>0.397</td>\n",
+       "      <td>0.568</td>\n",
+       "      <td>0.552</td>\n",
+       "      <td>0.861</td>\n",
+       "      <td>0.800</td>\n",
+       "      <td>0.4450</td>\n",
+       "      <td>0.4515</td>\n",
+       "      <td>0.302706</td>\n",
+       "      <td>0.318447</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>669</th>\n",
+       "      <td>big-run-sampled_full_filtered_no_dedup</td>\n",
+       "      <td>6</td>\n",
+       "      <td>165000</td>\n",
+       "      <td>0.469847</td>\n",
+       "      <td>0.426</td>\n",
+       "      <td>0.375</td>\n",
+       "      <td>0.472</td>\n",
+       "      <td>0.549</td>\n",
+       "      <td>0.234</td>\n",
+       "      <td>0.364</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.389</td>\n",
+       "      <td>0.401</td>\n",
+       "      <td>0.562</td>\n",
+       "      <td>0.548</td>\n",
+       "      <td>0.867</td>\n",
+       "      <td>0.795</td>\n",
+       "      <td>0.4435</td>\n",
+       "      <td>0.4475</td>\n",
+       "      <td>0.297586</td>\n",
+       "      <td>0.319279</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>670</th>\n",
+       "      <td>big-run-sampled_full_filtered_no_dedup</td>\n",
+       "      <td>6</td>\n",
+       "      <td>166000</td>\n",
+       "      <td>0.467651</td>\n",
+       "      <td>0.423</td>\n",
+       "      <td>0.365</td>\n",
+       "      <td>0.470</td>\n",
+       "      <td>0.555</td>\n",
+       "      <td>0.226</td>\n",
+       "      <td>0.356</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.392</td>\n",
+       "      <td>0.399</td>\n",
+       "      <td>0.564</td>\n",
+       "      <td>0.545</td>\n",
+       "      <td>0.872</td>\n",
+       "      <td>0.812</td>\n",
+       "      <td>0.4365</td>\n",
+       "      <td>0.4475</td>\n",
+       "      <td>0.297256</td>\n",
+       "      <td>0.319704</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>671</th>\n",
+       "      <td>big-run-sampled_full_filtered_no_dedup</td>\n",
+       "      <td>6</td>\n",
+       "      <td>167000</td>\n",
+       "      <td>0.469652</td>\n",
+       "      <td>0.416</td>\n",
+       "      <td>0.373</td>\n",
+       "      <td>0.469</td>\n",
+       "      <td>0.560</td>\n",
+       "      <td>0.234</td>\n",
+       "      <td>0.356</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.392</td>\n",
+       "      <td>0.394</td>\n",
+       "      <td>0.565</td>\n",
+       "      <td>0.557</td>\n",
+       "      <td>0.867</td>\n",
+       "      <td>0.803</td>\n",
+       "      <td>0.4430</td>\n",
+       "      <td>0.4455</td>\n",
+       "      <td>0.297409</td>\n",
+       "      <td>0.317717</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>672 rows × 22 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                    runname  seed   steps  agg_score  \\\n",
+       "0        big-run-sampled-fineweb-c4-filters     6       0   0.330893   \n",
+       "1        big-run-sampled-fineweb-c4-filters     6    1000   0.359303   \n",
+       "2        big-run-sampled-fineweb-c4-filters     6    2000   0.375393   \n",
+       "3        big-run-sampled-fineweb-c4-filters     6    3000   0.389655   \n",
+       "4        big-run-sampled-fineweb-c4-filters     6    4000   0.401195   \n",
+       "..                                      ...   ...     ...        ...   \n",
+       "667  big-run-sampled_full_filtered_no_dedup     6  163000   0.466255   \n",
+       "668  big-run-sampled_full_filtered_no_dedup     6  164000   0.469743   \n",
+       "669  big-run-sampled_full_filtered_no_dedup     6  165000   0.469847   \n",
+       "670  big-run-sampled_full_filtered_no_dedup     6  166000   0.467651   \n",
+       "671  big-run-sampled_full_filtered_no_dedup     6  167000   0.469652   \n",
+       "\n",
+       "     commonsense_qa/acc  commonsense_qa/acc_norm  hellaswag/acc  \\\n",
+       "0                 0.186                    0.233          0.272   \n",
+       "1                 0.250                    0.263          0.293   \n",
+       "2                 0.268                    0.277          0.319   \n",
+       "3                 0.303                    0.305          0.324   \n",
+       "4                 0.309                    0.310          0.353   \n",
+       "..                  ...                      ...            ...   \n",
+       "667               0.426                    0.372          0.469   \n",
+       "668               0.431                    0.376          0.467   \n",
+       "669               0.426                    0.375          0.472   \n",
+       "670               0.423                    0.365          0.470   \n",
+       "671               0.416                    0.373          0.469   \n",
+       "\n",
+       "     hellaswag/acc_norm  openbookqa/acc  openbookqa/acc_norm  ...  siqa/acc  \\\n",
+       "0                 0.258           0.166                0.286  ...     0.367   \n",
+       "1                 0.285           0.140                0.276  ...     0.376   \n",
+       "2                 0.324           0.150                0.274  ...     0.372   \n",
+       "3                 0.358           0.152                0.280  ...     0.383   \n",
+       "4                 0.393           0.138                0.288  ...     0.378   \n",
+       "..                  ...             ...                  ...  ...       ...   \n",
+       "667               0.555           0.242                0.354  ...     0.389   \n",
+       "668               0.556           0.232                0.356  ...     0.391   \n",
+       "669               0.549           0.234                0.364  ...     0.389   \n",
+       "670               0.555           0.226                0.356  ...     0.392   \n",
+       "671               0.560           0.234                0.356  ...     0.392   \n",
+       "\n",
+       "     siqa/acc_norm  winogrande/acc  winogrande/acc_norm  sciq/acc  \\\n",
+       "0            0.362           0.516                0.497     0.208   \n",
+       "1            0.401           0.497                0.479     0.594   \n",
+       "2            0.411           0.507                0.484     0.688   \n",
+       "3            0.389           0.520                0.506     0.741   \n",
+       "4            0.402           0.534                0.511     0.766   \n",
+       "..             ...             ...                  ...       ...   \n",
+       "667          0.394           0.563                0.544     0.869   \n",
+       "668          0.397           0.568                0.552     0.861   \n",
+       "669          0.401           0.562                0.548     0.867   \n",
+       "670          0.399           0.564                0.545     0.872   \n",
+       "671          0.394           0.565                0.557     0.867   \n",
+       "\n",
+       "     sciq/acc_norm  arc/acc  arc/acc_norm  mmlu/acc  mmlu/acc_norm  \n",
+       "0            0.202   0.2195        0.2510  0.230294       0.250147  \n",
+       "1            0.524   0.2740        0.2985  0.241617       0.251920  \n",
+       "2            0.606   0.3015        0.3270  0.246577       0.259146  \n",
+       "3            0.647   0.3395        0.3405  0.255001       0.268740  \n",
+       "4            0.652   0.3395        0.3495  0.256203       0.269056  \n",
+       "..             ...      ...           ...       ...            ...  \n",
+       "667          0.808   0.4460        0.4435  0.297125       0.317543  \n",
+       "668          0.800   0.4450        0.4515  0.302706       0.318447  \n",
+       "669          0.795   0.4435        0.4475  0.297586       0.319279  \n",
+       "670          0.812   0.4365        0.4475  0.297256       0.319704  \n",
+       "671          0.803   0.4430        0.4455  0.297409       0.317717  \n",
+       "\n",
+       "[672 rows x 22 columns]"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "from matplotlib.figure import Figure\n",
+    "\n",
+    "df = pd.read_csv(\"../src_data/all-filters-big-runs.csv\")\n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "839a06a71d9183e5",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-14T09:02:10.094329Z",
+     "start_time": "2024-05-14T09:02:10.081683Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['big-run-sampled-fineweb-c4-filters',\n",
+       " 'big-run-sampled_full_ind_minhash',\n",
+       " 'big-run-fineweb-v1-all-dumps',\n",
+       " 'big-run-sampled_full_filtered_no_dedup']"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "pd.unique(df[\"runname\"]).tolist()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "b610f43caefdf01",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-14T09:03:06.294766Z",
+     "start_time": "2024-05-14T09:03:06.291388Z"
+    },
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "runs_mapping = {\n",
+    "    # \"big-run-refinedweb\": \"RefinedWeb\",\n",
+    "    # \"big-run-c4\": \"C4\",\n",
+    "    \"big-run-sampled_full_filtered_no_dedup\": \"FineWeb: base filtering only\",\n",
+    "    \"big-run-sampled_full_ind_minhash\": \"FineWeb: independent MinHash (id mh)\",\n",
+    "    \"big-run-sampled-fineweb-c4-filters\": \"FineWeb: id mh + C4 filters\",\n",
+    "    \"big-run-fineweb-v1-all-dumps\": \"FineWeb: id mh + C4 + custom filters\",\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "initial_id",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-14T09:03:08.298110Z",
+     "start_time": "2024-05-14T09:03:08.024839Z"
+    },
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "from matplotlib import pyplot as plt\n",
+    "import os\n",
+    "import json\n",
+    "\n",
+    "metrics = ['agg_score', 'commonsense_qa/acc_norm', 'hellaswag/acc_norm', 'openbookqa/acc_norm', 'piqa/acc_norm',\n",
+    "                   'siqa/acc_norm', 'winogrande/acc_norm', 'arc/acc_norm', 'mmlu/acc_norm']\n",
+    "\n",
+    "def normalize_runname(runname):\n",
+    "    return runname.replace(\"/\", \"_\")\n",
+    "\n",
+    "grouped = (\n",
+    "    df.groupby([\"runname\", \"steps\"])\n",
+    "    .agg(\n",
+    "        {\n",
+    "            key: \"mean\" for key in metrics\n",
+    "        }\n",
+    "    )\n",
+    "    .reset_index()\n",
+    ")\n",
+    "\n",
+    "file_id=\"../assets/data/plots/all_filtering_steps\"\n",
+    "files = {}\n",
+    "for metric in metrics:\n",
+    "    datas = {}\n",
+    "    for name, group in grouped.groupby(\"runname\"):\n",
+    "        group = group[[\"steps\", metric]].sort_values(by=\"steps\")\n",
+    "        group = group.set_index(\"steps\")\n",
+    "        rolling_avg = group\n",
+    "        # rolling_avg = group.rolling(window=5).mean()\n",
+    "        datas[name] = {\n",
+    "            \"x\": (rolling_avg.index * 2048 * 1024 * 1e-9).tolist(),\n",
+    "            \"y\": rolling_avg[metric].tolist(),\n",
+    "            \"label\": runs_mapping[name],\n",
+    "        }\n",
+    "    # Sort the datata based on the steps\n",
+    "    datas = {k: v for k, v in sorted(datas.items(), key=lambda x: -x[1][\"y\"][-1])}\n",
+    "    # Create a folder\n",
+    "    os.makedirs(f\"{file_id}\", exist_ok=True)\n",
+    "    with open(f\"{file_id}/{normalize_runname(metric)}.json\", \"w\") as f:\n",
+    "        json.dump({\n",
+    "            \"data\": datas,\n",
+    "            \"layout\": {\n",
+    "                \"title\": {\n",
+    "                    \"text\": \"The different FineWeb processing steps\"\n",
+    "                },\n",
+    "            }\n",
+    "        }, f)\n",
+    "    files[metric] = {\"file\": f\"{normalize_runname(metric)}.json\"}\n",
+    "# Create l\n",
+    "with open(f\"{file_id}/index.json\", \"w\") as f:\n",
+    "    json.dump({\n",
+    "        \"files\": files,\n",
+    "        \"settings\": {\n",
+    "            \"defaultMetric\": \"agg_score\",\n",
+    "            \"slider\":{\"min\":0,\"max\":30,\"default\":5}\n",
+    "        }\n",
+    "    }, f)\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "af28ebbd054cdc33",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-14T08:14:41.132508Z",
+     "start_time": "2024-05-14T08:14:41.130025Z"
+    },
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6b8c428e2fedeb1a",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/plot_c4_filters_hellaswag.ipynb

@@ -0,0 +1,580 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "138889b92720ce2e",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-13T14:36:31.336129Z",
+     "start_time": "2024-05-13T14:36:31.323847Z"
+    },
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>runname</th>\n",
+       "      <th>seed</th>\n",
+       "      <th>steps</th>\n",
+       "      <th>agg_score</th>\n",
+       "      <th>commonsense_qa/acc</th>\n",
+       "      <th>commonsense_qa/acc_norm</th>\n",
+       "      <th>hellaswag/acc</th>\n",
+       "      <th>hellaswag/acc_norm</th>\n",
+       "      <th>openbookqa/acc</th>\n",
+       "      <th>openbookqa/acc_norm</th>\n",
+       "      <th>...</th>\n",
+       "      <th>siqa/acc</th>\n",
+       "      <th>siqa/acc_norm</th>\n",
+       "      <th>winogrande/acc</th>\n",
+       "      <th>winogrande/acc_norm</th>\n",
+       "      <th>sciq/acc</th>\n",
+       "      <th>sciq/acc_norm</th>\n",
+       "      <th>arc/acc</th>\n",
+       "      <th>arc/acc_norm</th>\n",
+       "      <th>mmlu/acc</th>\n",
+       "      <th>mmlu/acc_norm</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>filtering-baseline-2019-18-40gt</td>\n",
+       "      <td>5</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.330953</td>\n",
+       "      <td>0.186</td>\n",
+       "      <td>0.233</td>\n",
+       "      <td>0.272</td>\n",
+       "      <td>0.258</td>\n",
+       "      <td>0.166</td>\n",
+       "      <td>0.286</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.367</td>\n",
+       "      <td>0.362</td>\n",
+       "      <td>0.516</td>\n",
+       "      <td>0.497</td>\n",
+       "      <td>0.210</td>\n",
+       "      <td>0.202</td>\n",
+       "      <td>0.2190</td>\n",
+       "      <td>0.2515</td>\n",
+       "      <td>0.230285</td>\n",
+       "      <td>0.250127</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>filtering-baseline-2019-18-40gt</td>\n",
+       "      <td>5</td>\n",
+       "      <td>1000</td>\n",
+       "      <td>0.357474</td>\n",
+       "      <td>0.239</td>\n",
+       "      <td>0.271</td>\n",
+       "      <td>0.297</td>\n",
+       "      <td>0.287</td>\n",
+       "      <td>0.146</td>\n",
+       "      <td>0.260</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.365</td>\n",
+       "      <td>0.396</td>\n",
+       "      <td>0.503</td>\n",
+       "      <td>0.486</td>\n",
+       "      <td>0.568</td>\n",
+       "      <td>0.502</td>\n",
+       "      <td>0.2665</td>\n",
+       "      <td>0.2855</td>\n",
+       "      <td>0.242526</td>\n",
+       "      <td>0.253291</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>filtering-baseline-2019-18-40gt</td>\n",
+       "      <td>5</td>\n",
+       "      <td>2000</td>\n",
+       "      <td>0.377436</td>\n",
+       "      <td>0.280</td>\n",
+       "      <td>0.284</td>\n",
+       "      <td>0.321</td>\n",
+       "      <td>0.332</td>\n",
+       "      <td>0.134</td>\n",
+       "      <td>0.268</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.368</td>\n",
+       "      <td>0.399</td>\n",
+       "      <td>0.519</td>\n",
+       "      <td>0.502</td>\n",
+       "      <td>0.686</td>\n",
+       "      <td>0.590</td>\n",
+       "      <td>0.3030</td>\n",
+       "      <td>0.3215</td>\n",
+       "      <td>0.245745</td>\n",
+       "      <td>0.260988</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>filtering-baseline-2019-18-40gt</td>\n",
+       "      <td>5</td>\n",
+       "      <td>3000</td>\n",
+       "      <td>0.387994</td>\n",
+       "      <td>0.277</td>\n",
+       "      <td>0.291</td>\n",
+       "      <td>0.339</td>\n",
+       "      <td>0.359</td>\n",
+       "      <td>0.132</td>\n",
+       "      <td>0.280</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.394</td>\n",
+       "      <td>0.404</td>\n",
+       "      <td>0.520</td>\n",
+       "      <td>0.503</td>\n",
+       "      <td>0.721</td>\n",
+       "      <td>0.622</td>\n",
+       "      <td>0.3210</td>\n",
+       "      <td>0.3385</td>\n",
+       "      <td>0.250427</td>\n",
+       "      <td>0.264451</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>filtering-baseline-2019-18-40gt</td>\n",
+       "      <td>5</td>\n",
+       "      <td>4000</td>\n",
+       "      <td>0.396110</td>\n",
+       "      <td>0.299</td>\n",
+       "      <td>0.315</td>\n",
+       "      <td>0.340</td>\n",
+       "      <td>0.366</td>\n",
+       "      <td>0.158</td>\n",
+       "      <td>0.286</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.376</td>\n",
+       "      <td>0.399</td>\n",
+       "      <td>0.515</td>\n",
+       "      <td>0.500</td>\n",
+       "      <td>0.739</td>\n",
+       "      <td>0.620</td>\n",
+       "      <td>0.3320</td>\n",
+       "      <td>0.3445</td>\n",
+       "      <td>0.256134</td>\n",
+       "      <td>0.270382</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>250</th>\n",
+       "      <td>sm-baseline-c4</td>\n",
+       "      <td>6</td>\n",
+       "      <td>10000</td>\n",
+       "      <td>0.430443</td>\n",
+       "      <td>0.335</td>\n",
+       "      <td>0.326</td>\n",
+       "      <td>0.379</td>\n",
+       "      <td>0.474</td>\n",
+       "      <td>0.176</td>\n",
+       "      <td>0.340</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.385</td>\n",
+       "      <td>0.406</td>\n",
+       "      <td>0.525</td>\n",
+       "      <td>0.523</td>\n",
+       "      <td>0.767</td>\n",
+       "      <td>0.675</td>\n",
+       "      <td>0.3765</td>\n",
+       "      <td>0.3750</td>\n",
+       "      <td>0.269139</td>\n",
+       "      <td>0.280545</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>251</th>\n",
+       "      <td>sm-baseline-c4</td>\n",
+       "      <td>6</td>\n",
+       "      <td>11000</td>\n",
+       "      <td>0.430776</td>\n",
+       "      <td>0.341</td>\n",
+       "      <td>0.323</td>\n",
+       "      <td>0.391</td>\n",
+       "      <td>0.481</td>\n",
+       "      <td>0.192</td>\n",
+       "      <td>0.346</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.390</td>\n",
+       "      <td>0.405</td>\n",
+       "      <td>0.531</td>\n",
+       "      <td>0.515</td>\n",
+       "      <td>0.766</td>\n",
+       "      <td>0.676</td>\n",
+       "      <td>0.3775</td>\n",
+       "      <td>0.3770</td>\n",
+       "      <td>0.266895</td>\n",
+       "      <td>0.281210</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>252</th>\n",
+       "      <td>sm-baseline-c4</td>\n",
+       "      <td>6</td>\n",
+       "      <td>12000</td>\n",
+       "      <td>0.430352</td>\n",
+       "      <td>0.340</td>\n",
+       "      <td>0.319</td>\n",
+       "      <td>0.392</td>\n",
+       "      <td>0.475</td>\n",
+       "      <td>0.192</td>\n",
+       "      <td>0.342</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.377</td>\n",
+       "      <td>0.395</td>\n",
+       "      <td>0.528</td>\n",
+       "      <td>0.518</td>\n",
+       "      <td>0.785</td>\n",
+       "      <td>0.688</td>\n",
+       "      <td>0.3755</td>\n",
+       "      <td>0.3840</td>\n",
+       "      <td>0.267159</td>\n",
+       "      <td>0.279819</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>253</th>\n",
+       "      <td>sm-baseline-c4</td>\n",
+       "      <td>6</td>\n",
+       "      <td>13000</td>\n",
+       "      <td>0.432136</td>\n",
+       "      <td>0.339</td>\n",
+       "      <td>0.326</td>\n",
+       "      <td>0.395</td>\n",
+       "      <td>0.477</td>\n",
+       "      <td>0.198</td>\n",
+       "      <td>0.348</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.390</td>\n",
+       "      <td>0.405</td>\n",
+       "      <td>0.529</td>\n",
+       "      <td>0.518</td>\n",
+       "      <td>0.785</td>\n",
+       "      <td>0.682</td>\n",
+       "      <td>0.3780</td>\n",
+       "      <td>0.3825</td>\n",
+       "      <td>0.269719</td>\n",
+       "      <td>0.281585</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>254</th>\n",
+       "      <td>sm-baseline-c4</td>\n",
+       "      <td>6</td>\n",
+       "      <td>13500</td>\n",
+       "      <td>0.433866</td>\n",
+       "      <td>0.344</td>\n",
+       "      <td>0.328</td>\n",
+       "      <td>0.394</td>\n",
+       "      <td>0.484</td>\n",
+       "      <td>0.198</td>\n",
+       "      <td>0.334</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.388</td>\n",
+       "      <td>0.406</td>\n",
+       "      <td>0.531</td>\n",
+       "      <td>0.523</td>\n",
+       "      <td>0.778</td>\n",
+       "      <td>0.682</td>\n",
+       "      <td>0.3795</td>\n",
+       "      <td>0.3845</td>\n",
+       "      <td>0.269601</td>\n",
+       "      <td>0.284425</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>255 rows × 22 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                             runname  seed  steps  agg_score  \\\n",
+       "0    filtering-baseline-2019-18-40gt     5      0   0.330953   \n",
+       "1    filtering-baseline-2019-18-40gt     5   1000   0.357474   \n",
+       "2    filtering-baseline-2019-18-40gt     5   2000   0.377436   \n",
+       "3    filtering-baseline-2019-18-40gt     5   3000   0.387994   \n",
+       "4    filtering-baseline-2019-18-40gt     5   4000   0.396110   \n",
+       "..                               ...   ...    ...        ...   \n",
+       "250                   sm-baseline-c4     6  10000   0.430443   \n",
+       "251                   sm-baseline-c4     6  11000   0.430776   \n",
+       "252                   sm-baseline-c4     6  12000   0.430352   \n",
+       "253                   sm-baseline-c4     6  13000   0.432136   \n",
+       "254                   sm-baseline-c4     6  13500   0.433866   \n",
+       "\n",
+       "     commonsense_qa/acc  commonsense_qa/acc_norm  hellaswag/acc  \\\n",
+       "0                 0.186                    0.233          0.272   \n",
+       "1                 0.239                    0.271          0.297   \n",
+       "2                 0.280                    0.284          0.321   \n",
+       "3                 0.277                    0.291          0.339   \n",
+       "4                 0.299                    0.315          0.340   \n",
+       "..                  ...                      ...            ...   \n",
+       "250               0.335                    0.326          0.379   \n",
+       "251               0.341                    0.323          0.391   \n",
+       "252               0.340                    0.319          0.392   \n",
+       "253               0.339                    0.326          0.395   \n",
+       "254               0.344                    0.328          0.394   \n",
+       "\n",
+       "     hellaswag/acc_norm  openbookqa/acc  openbookqa/acc_norm  ...  siqa/acc  \\\n",
+       "0                 0.258           0.166                0.286  ...     0.367   \n",
+       "1                 0.287           0.146                0.260  ...     0.365   \n",
+       "2                 0.332           0.134                0.268  ...     0.368   \n",
+       "3                 0.359           0.132                0.280  ...     0.394   \n",
+       "4                 0.366           0.158                0.286  ...     0.376   \n",
+       "..                  ...             ...                  ...  ...       ...   \n",
+       "250               0.474           0.176                0.340  ...     0.385   \n",
+       "251               0.481           0.192                0.346  ...     0.390   \n",
+       "252               0.475           0.192                0.342  ...     0.377   \n",
+       "253               0.477           0.198                0.348  ...     0.390   \n",
+       "254               0.484           0.198                0.334  ...     0.388   \n",
+       "\n",
+       "     siqa/acc_norm  winogrande/acc  winogrande/acc_norm  sciq/acc  \\\n",
+       "0            0.362           0.516                0.497     0.210   \n",
+       "1            0.396           0.503                0.486     0.568   \n",
+       "2            0.399           0.519                0.502     0.686   \n",
+       "3            0.404           0.520                0.503     0.721   \n",
+       "4            0.399           0.515                0.500     0.739   \n",
+       "..             ...             ...                  ...       ...   \n",
+       "250          0.406           0.525                0.523     0.767   \n",
+       "251          0.405           0.531                0.515     0.766   \n",
+       "252          0.395           0.528                0.518     0.785   \n",
+       "253          0.405           0.529                0.518     0.785   \n",
+       "254          0.406           0.531                0.523     0.778   \n",
+       "\n",
+       "     sciq/acc_norm  arc/acc  arc/acc_norm  mmlu/acc  mmlu/acc_norm  \n",
+       "0            0.202   0.2190        0.2515  0.230285       0.250127  \n",
+       "1            0.502   0.2665        0.2855  0.242526       0.253291  \n",
+       "2            0.590   0.3030        0.3215  0.245745       0.260988  \n",
+       "3            0.622   0.3210        0.3385  0.250427       0.264451  \n",
+       "4            0.620   0.3320        0.3445  0.256134       0.270382  \n",
+       "..             ...      ...           ...       ...            ...  \n",
+       "250          0.675   0.3765        0.3750  0.269139       0.280545  \n",
+       "251          0.676   0.3775        0.3770  0.266895       0.281210  \n",
+       "252          0.688   0.3755        0.3840  0.267159       0.279819  \n",
+       "253          0.682   0.3780        0.3825  0.269719       0.281585  \n",
+       "254          0.682   0.3795        0.3845  0.269601       0.284425  \n",
+       "\n",
+       "[255 rows x 22 columns]"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "from matplotlib.figure import Figure\n",
+    "\n",
+    "df = pd.read_csv(\"../src_data/c4-filters.csv\")\n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "839a06a71d9183e5",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-13T14:36:32.338012Z",
+     "start_time": "2024-05-13T14:36:32.335209Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['filtering-baseline-2019-18-40gt',\n",
+       " 'filtering-baseline-2019-18-60gt',\n",
+       " 'filtering-c4-all-except-terminal_punct',\n",
+       " 'filtering-c4-all',\n",
+       " 'filtering-c4-curly_bracket',\n",
+       " 'filtering-c4-terminal_punct',\n",
+       " 'filtering-c4-word_lengths',\n",
+       " 'sm-baseline-c4']"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "pd.unique(df[\"runname\"]).tolist()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "b610f43caefdf01",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-13T16:06:36.968532Z",
+     "start_time": "2024-05-13T16:06:36.966172Z"
+    },
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "runs_mapping = {\n",
+    " # 'filtering-baseline-2019-18-40gt': \"baseline\",\n",
+    " 'filtering-baseline-2019-18-60gt': \"baseline\",\n",
+    " 'filtering-c4-curly_bracket': \"curly_bracket filter\",\n",
+    " 'filtering-c4-terminal_punct': \"terminal_punct filter\",\n",
+    " 'filtering-c4-word_lengths': \"word_lengths filter\",\n",
+    " 'filtering-c4-all': \"All filters\",\n",
+    " 'filtering-c4-all-except-terminal_punct': \"All filters except terminal_punct\",\n",
+    " 'sm-baseline-c4': \"C4\"\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "initial_id",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-13T16:06:37.459935Z",
+     "start_time": "2024-05-13T16:06:37.181024Z"
+    },
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "from matplotlib import pyplot as plt\n",
+    "\n",
+    "\n",
+    "import json\n",
+    "import os\n",
+    "from matplotlib import pyplot as plt\n",
+    "metrics = ['agg_score', 'commonsense_qa/acc_norm', 'hellaswag/acc_norm', 'openbookqa/acc_norm', 'piqa/acc_norm',\n",
+    "                   'siqa/acc_norm', 'winogrande/acc_norm', 'arc/acc_norm', 'mmlu/acc_norm']\n",
+    "\n",
+    "def normalize_runname(runname):\n",
+    "    return runname.replace(\"/\", \"_\")\n",
+    "\n",
+    "grouped = (\n",
+    "    df.groupby([\"runname\", \"steps\"])\n",
+    "    .agg(\n",
+    "        {\n",
+    "            key: \"mean\" for key in metrics\n",
+    "        }\n",
+    "    )\n",
+    "    .reset_index()\n",
+    ")\n",
+    "\n",
+    "file_id=\"../assets/data/plots/c4_filters_hellaswag\"\n",
+    "files = {}\n",
+    "for metric in metrics:\n",
+    "    datas = {}\n",
+    "    for name, group in grouped.groupby(\"runname\"):\n",
+    "        if name not in runs_mapping:\n",
+    "            continue\n",
+    "        group = group[[\"steps\", metric]].sort_values(by=\"steps\")\n",
+    "        group = group.set_index(\"steps\")\n",
+    "        rolling_avg = group\n",
+    "        datas[name] = {\n",
+    "            \"x\": (rolling_avg.index * 2048 * 1024 * 1e-9).tolist(),\n",
+    "            \"y\": rolling_avg[metric].tolist(),\n",
+    "            \"label\": runs_mapping[name],\n",
+    "        }\n",
+    "    # Sort the datata based on the steps\n",
+    "    datas = {k: v for k, v in sorted(datas.items(), key=lambda x: -x[1][\"y\"][-1])}\n",
+    "    # Create a folder\n",
+    "    os.makedirs(f\"{file_id}\", exist_ok=True)\n",
+    "    with open(f\"{file_id}/{normalize_runname(metric)}.json\", \"w\") as f:\n",
+    "        json.dump({\n",
+    "            \"data\": datas,\n",
+    "            \"layout\": {\n",
+    "                \"title\": {\n",
+    "                    \"text\": \"C4 filtering effect on HellaSwag\"\n",
+    "                },\n",
+    "            }\n",
+    "        }, f)\n",
+    "    files[metric] = {\"file\": f\"{normalize_runname(metric)}.json\"}\n",
+    "# Create index\n",
+    "with open(f\"{file_id}/index.json\", \"w\") as f:\n",
+    "    json.dump({\n",
+    "        \"files\": files,\n",
+    "        \"settings\": {\n",
+    "            \"defaultMetric\": \"hellaswag/acc_norm\",\n",
+    "            \"slider\":{\"min\":0,\"max\":10,\"default\":3}\n",
+    "        }\n",
+    "    }, f)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "af28ebbd054cdc33",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-04-30T12:52:05.836260Z",
+     "start_time": "2024-04-30T12:52:05.834381Z"
+    },
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/plot_commoncrawl_dumps.ipynb

@@ -0,0 +1,284 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "id": "initial_id",
+   "metadata": {
+    "collapsed": true,
+    "ExecuteTime": {
+     "end_time": "2024-05-14T09:57:03.097798Z",
+     "start_time": "2024-05-14T09:57:02.853658Z"
+    }
+   },
+   "source": [
+    "import pandas as pd"
+   ],
+   "execution_count": 2,
+   "outputs": []
+  },
+  {
+   "cell_type": "code",
+   "source": [
+    "df = pd.read_csv(\"/home/gui/hf_dev/datatrove/blogpost/data/commoncrawl_dumps.csv\")"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "ExecuteTime": {
+     "end_time": "2024-05-14T09:57:03.110303Z",
+     "start_time": "2024-05-14T09:57:03.098988Z"
+    }
+   },
+   "id": "157e18836c20793c",
+   "execution_count": 3,
+   "outputs": []
+  },
+  {
+   "cell_type": "code",
+   "source": [
+    "grouped = df.groupby('runname')\n",
+    "\n",
+    "# Define a function to take the top 6 rows of each group\n",
+    "def top_6_avg(group):\n",
+    "    # Sort the group by \"steps\" in descending order\n",
+    "    sorted_group = group.sort_values(by='steps', ascending=False)\n",
+    "    # Take the top 6 rows\n",
+    "    top_6 = sorted_group.head(6)\n",
+    "    # Calculate the average of \"agg_score\"\n",
+    "    avg_score = top_6['agg_score'].mean()\n",
+    "    return avg_score\n",
+    "\n",
+    "def top_6_stats(group):\n",
+    "    # Sort the group by \"steps\" in descending order\n",
+    "    sorted_group = group.sort_values(by='steps', ascending=False)\n",
+    "    # Take the top 6 rows\n",
+    "    top_6 = sorted_group.head(6)\n",
+    "    # Calculate the average of \"agg_score\"\n",
+    "    avg_score = top_6['agg_score'].mean()\n",
+    "    # Calculate the standard deviation of \"agg_score\"\n",
+    "    std_dev = top_6['agg_score'].std()\n",
+    "    return pd.Series({'avg': avg_score, 'std_dev': std_dev})\n",
+    "\n",
+    "# Apply the function to each group and aggregate the results\n",
+    "result = grouped.apply(top_6_stats)"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "ExecuteTime": {
+     "end_time": "2024-05-14T09:57:03.227764Z",
+     "start_time": "2024-05-14T09:57:03.183929Z"
+    }
+   },
+   "id": "af7c0416a6371f9a",
+   "execution_count": 4,
+   "outputs": []
+  },
+  {
+   "cell_type": "code",
+   "source": [
+    "result"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "ExecuteTime": {
+     "end_time": "2024-05-14T09:57:03.784515Z",
+     "start_time": "2024-05-14T09:57:03.775829Z"
+    }
+   },
+   "id": "65c0cd58c6f9f9d6",
+   "execution_count": 5,
+   "outputs": []
+  },
+  {
+   "cell_type": "code",
+   "source": [
+    "import numpy as np\n",
+    "import matplotlib\n",
+    "import matplotlib.pyplot as plt\n",
+    "import matplotlib.colors as mcolors\n",
+    "\n",
+    "# Assuming you have already computed the result DataFrame\n",
+    "\n",
+    "# Sort the result DataFrame by \"runname\"\n",
+    "result_sorted = result.sort_index()\n",
+    "colors = result_sorted.index.str.split('-').str[0].astype(int)\n",
+    "\n",
+    "cmap = plt.cm.tab10\n",
+    "\n",
+    "# Create a new colormap without transparency\n",
+    "new_colors = cmap(np.linspace(0, 1, cmap.N))\n",
+    "new_colors = np.concatenate((new_colors[-2:], new_colors))\n",
+    "new_cmap = mcolors.ListedColormap(new_colors)\n",
+    "rgba_colors = new_cmap(new_colors)\n",
+    "\n",
+    "\n",
+    "# Plotting\n",
+    "plt.figure(figsize=(15, 10))\n",
+    "# Join the points with a line\n",
+    "plt.plot(range(len(result_sorted)), result_sorted[\"avg\"], linestyle='-', color='gray', alpha=0.5, zorder=1)\n",
+    "scatter = plt.scatter(range(len(result_sorted)), result_sorted[\"avg\"], c=colors, cmap=new_cmap, marker='o', s=100, zorder=2)\n",
+    "\n",
+    "norm = plt.Normalize(min(colors), max(colors))\n",
+    "\n",
+    "import matplotlib.cm as cm\n",
+    "# Creating a ScalarMappable object with the tab20 colormap and normalization\n",
+    "sm = cm.ScalarMappable(cmap=new_cmap, norm=norm)\n",
+    "\n",
+    "plt.xlabel('Year', fontsize=18)\n",
+    "plt.ylabel('Average Agg Score', fontsize=18)\n",
+    "plt.title('Score by dump', fontsize=24)\n",
+    "plt.xticks(range(len(result_sorted)), colors, ha='center', fontsize=14)\n",
+    "plt.yticks(fontsize=14)\n",
+    "ax = plt.gca()\n",
+    "\n",
+    "# for i in range(len(result_sorted)):\n",
+    "#     plt.errorbar(i, result_sorted.iloc[i]['avg'], yerr=result_sorted.iloc[i]['std_dev'], fmt='o', color=sm.to_rgba(colors[i]), markersize=0, capsize=5)\n",
+    "prev = None\n",
+    "labels = ax.xaxis.get_ticklabels()\n",
+    "# labels[0].set_horizontalalignment('right')\n",
+    "lines = []\n",
+    "for x, name in enumerate(colors.tolist()):\n",
+    "    if name != prev:\n",
+    "        plt.axvline(x=x, color='grey', linestyle=':')\n",
+    "        lines.append(x)\n",
+    "    prev = name\n",
+    "\n",
+    "mids = np.floor((np.array(lines[:-1]) + np.array(lines[1:])) / 2)\n",
+    "for x in range(len(colors) - 1):\n",
+    "    if x not in mids:\n",
+    "        labels[x].set_visible(False)\n",
+    "labels[-1].set_horizontalalignment('left')\n",
+    "    \n",
+    "\n",
+    "# plt.grid(True)\n",
+    "plt.savefig(\"/home/gui/hf_dev/datatrove/blogpost/plots/score_by_dump.png\", bbox_inches='tight', dpi=300)\n",
+    "plt.show()"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "ExecuteTime": {
+     "end_time": "2024-05-14T12:33:41.469562Z",
+     "start_time": "2024-05-14T12:33:40.411105Z"
+    }
+   },
+   "id": "412ed6b4570d10e9",
+   "execution_count": 98,
+   "outputs": []
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-14T12:18:06.365519Z",
+     "start_time": "2024-05-14T12:18:06.360995Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "    \n",
+    "new_colors = cmap(np.linspace(0, 1, cmap.N))\n",
+    "new_colors = np.concatenate((new_colors[-2:], new_colors))\n",
+    "mcolors.ListedColormap(new_colors)"
+   ],
+   "id": "270bd97983706aee",
+   "execution_count": 85,
+   "outputs": []
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-14T12:13:03.523524Z",
+     "start_time": "2024-05-14T12:13:03.518910Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "new_cmap",
+   "id": "ae52ddd47cf306a1",
+   "execution_count": 76,
+   "outputs": []
+  },
+  {
+   "metadata": {},
+   "cell_type": "markdown",
+   "source": "Flipped axis",
+   "id": "dd4bbdf230df5953"
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-14T10:16:00.731056Z",
+     "start_time": "2024-05-14T10:15:59.648467Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "import matplotlib.pyplot as plt\n",
+    "\n",
+    "# Assuming you have already computed the result DataFrame\n",
+    "\n",
+    "# Sort the result DataFrame by \"runname\"\n",
+    "result_sorted = result.sort_index()\n",
+    "colors = result_sorted.index.str.split('-').str[0].astype(int)\n",
+    "\n",
+    "rgba_colors = plt.cm.tab20(colors)\n",
+    "# Plotting\n",
+    "plt.figure(figsize=(10, 20))\n",
+    "scatter = plt.scatter(result_sorted[\"avg\"], range(len(result_sorted)), c=colors, cmap='tab20', marker='o', s=100)\n",
+    "# Join the points with a line\n",
+    "plt.plot(result_sorted[\"avg\"], range(len(result_sorted)), linestyle='-', color='gray', alpha=0.5)\n",
+    "\n",
+    "norm = plt.Normalize(min(colors), max(colors))\n",
+    "\n",
+    "import matplotlib.cm as cm\n",
+    "\n",
+    "# Creating a ScalarMappable object with the tab20 colormap and normalization\n",
+    "sm = cm.ScalarMappable(cmap='tab20', norm=norm)\n",
+    "\n",
+    "plt.xlabel('Dump')\n",
+    "plt.ylabel('Average Agg Score')\n",
+    "plt.title('Score by dump. 3 last checkpoints of each seed avgd')\n",
+    "plt.yticks(range(len(result_sorted)), result_sorted.index, ha='right', rotation_mode='anchor')\n",
+    "ax = plt.gca()\n",
+    "\n",
+    "# for i in range(len(result_sorted)):\n",
+    "#     plt.errorbar(i, result_sorted.iloc[i]['avg'], yerr=result_sorted.iloc[i]['std_dev'], fmt='o', color=sm.to_rgba(colors[i]), markersize=0, capsize=5)\n",
+    "# for label in ax.xaxis.get_ticklabels()[1::2]:\n",
+    "#     label.set_visible(False)\n",
+    "\n",
+    "plt.grid(True)\n",
+    "plt.savefig(\"/home/gui/hf_dev/datatrove/blogpost/plots/score_by_dump.png\", bbox_inches='tight', dpi=300)\n",
+    "plt.show()\n"
+   ],
+   "id": "49656c68704a55ca",
+   "execution_count": 36,
+   "outputs": []
+  },
+  {
+   "metadata": {},
+   "cell_type": "code",
+   "execution_count": null,
+   "source": "",
+   "id": "1872a68fa04b776d",
+   "outputs": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/plot_custom_filters.ipynb

@@ -0,0 +1,534 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "138889b92720ce2e",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-14T09:06:04.487186Z",
+     "start_time": "2024-05-14T09:06:04.255111Z"
+    },
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>runname</th>\n",
+       "      <th>seed</th>\n",
+       "      <th>steps</th>\n",
+       "      <th>agg_score</th>\n",
+       "      <th>commonsense_qa/acc</th>\n",
+       "      <th>commonsense_qa/acc_norm</th>\n",
+       "      <th>hellaswag/acc</th>\n",
+       "      <th>hellaswag/acc_norm</th>\n",
+       "      <th>openbookqa/acc</th>\n",
+       "      <th>openbookqa/acc_norm</th>\n",
+       "      <th>...</th>\n",
+       "      <th>siqa/acc</th>\n",
+       "      <th>siqa/acc_norm</th>\n",
+       "      <th>winogrande/acc</th>\n",
+       "      <th>winogrande/acc_norm</th>\n",
+       "      <th>sciq/acc</th>\n",
+       "      <th>sciq/acc_norm</th>\n",
+       "      <th>arc/acc</th>\n",
+       "      <th>arc/acc_norm</th>\n",
+       "      <th>mmlu/acc</th>\n",
+       "      <th>mmlu/acc_norm</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>filtering-baseline-2019-18-40gt</td>\n",
+       "      <td>5</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.330953</td>\n",
+       "      <td>0.186</td>\n",
+       "      <td>0.233</td>\n",
+       "      <td>0.272</td>\n",
+       "      <td>0.258</td>\n",
+       "      <td>0.166</td>\n",
+       "      <td>0.286</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.367</td>\n",
+       "      <td>0.362</td>\n",
+       "      <td>0.516</td>\n",
+       "      <td>0.497</td>\n",
+       "      <td>0.210</td>\n",
+       "      <td>0.202</td>\n",
+       "      <td>0.2190</td>\n",
+       "      <td>0.2515</td>\n",
+       "      <td>0.230285</td>\n",
+       "      <td>0.250127</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>filtering-baseline-2019-18-40gt</td>\n",
+       "      <td>5</td>\n",
+       "      <td>1000</td>\n",
+       "      <td>0.357474</td>\n",
+       "      <td>0.239</td>\n",
+       "      <td>0.271</td>\n",
+       "      <td>0.297</td>\n",
+       "      <td>0.287</td>\n",
+       "      <td>0.146</td>\n",
+       "      <td>0.260</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.365</td>\n",
+       "      <td>0.396</td>\n",
+       "      <td>0.503</td>\n",
+       "      <td>0.486</td>\n",
+       "      <td>0.568</td>\n",
+       "      <td>0.502</td>\n",
+       "      <td>0.2665</td>\n",
+       "      <td>0.2855</td>\n",
+       "      <td>0.242526</td>\n",
+       "      <td>0.253291</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>filtering-baseline-2019-18-40gt</td>\n",
+       "      <td>5</td>\n",
+       "      <td>2000</td>\n",
+       "      <td>0.377436</td>\n",
+       "      <td>0.280</td>\n",
+       "      <td>0.284</td>\n",
+       "      <td>0.321</td>\n",
+       "      <td>0.332</td>\n",
+       "      <td>0.134</td>\n",
+       "      <td>0.268</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.368</td>\n",
+       "      <td>0.399</td>\n",
+       "      <td>0.519</td>\n",
+       "      <td>0.502</td>\n",
+       "      <td>0.686</td>\n",
+       "      <td>0.590</td>\n",
+       "      <td>0.3030</td>\n",
+       "      <td>0.3215</td>\n",
+       "      <td>0.245745</td>\n",
+       "      <td>0.260988</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>filtering-baseline-2019-18-40gt</td>\n",
+       "      <td>5</td>\n",
+       "      <td>3000</td>\n",
+       "      <td>0.387994</td>\n",
+       "      <td>0.277</td>\n",
+       "      <td>0.291</td>\n",
+       "      <td>0.339</td>\n",
+       "      <td>0.359</td>\n",
+       "      <td>0.132</td>\n",
+       "      <td>0.280</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.394</td>\n",
+       "      <td>0.404</td>\n",
+       "      <td>0.520</td>\n",
+       "      <td>0.503</td>\n",
+       "      <td>0.721</td>\n",
+       "      <td>0.622</td>\n",
+       "      <td>0.3210</td>\n",
+       "      <td>0.3385</td>\n",
+       "      <td>0.250427</td>\n",
+       "      <td>0.264451</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>filtering-baseline-2019-18-40gt</td>\n",
+       "      <td>5</td>\n",
+       "      <td>4000</td>\n",
+       "      <td>0.396110</td>\n",
+       "      <td>0.299</td>\n",
+       "      <td>0.315</td>\n",
+       "      <td>0.340</td>\n",
+       "      <td>0.366</td>\n",
+       "      <td>0.158</td>\n",
+       "      <td>0.286</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.376</td>\n",
+       "      <td>0.399</td>\n",
+       "      <td>0.515</td>\n",
+       "      <td>0.500</td>\n",
+       "      <td>0.739</td>\n",
+       "      <td>0.620</td>\n",
+       "      <td>0.3320</td>\n",
+       "      <td>0.3445</td>\n",
+       "      <td>0.256134</td>\n",
+       "      <td>0.270382</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>129</th>\n",
+       "      <td>filtering-custom-short-line-ratio-0.67</td>\n",
+       "      <td>6</td>\n",
+       "      <td>10000</td>\n",
+       "      <td>0.422300</td>\n",
+       "      <td>0.333</td>\n",
+       "      <td>0.341</td>\n",
+       "      <td>0.382</td>\n",
+       "      <td>0.417</td>\n",
+       "      <td>0.192</td>\n",
+       "      <td>0.318</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.389</td>\n",
+       "      <td>0.407</td>\n",
+       "      <td>0.536</td>\n",
+       "      <td>0.530</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.3630</td>\n",
+       "      <td>0.3700</td>\n",
+       "      <td>0.266752</td>\n",
+       "      <td>0.284400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>130</th>\n",
+       "      <td>filtering-custom-short-line-ratio-0.67</td>\n",
+       "      <td>6</td>\n",
+       "      <td>11000</td>\n",
+       "      <td>0.425840</td>\n",
+       "      <td>0.345</td>\n",
+       "      <td>0.340</td>\n",
+       "      <td>0.395</td>\n",
+       "      <td>0.432</td>\n",
+       "      <td>0.192</td>\n",
+       "      <td>0.322</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.379</td>\n",
+       "      <td>0.405</td>\n",
+       "      <td>0.527</td>\n",
+       "      <td>0.531</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.3680</td>\n",
+       "      <td>0.3745</td>\n",
+       "      <td>0.267998</td>\n",
+       "      <td>0.282222</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>131</th>\n",
+       "      <td>filtering-custom-short-line-ratio-0.67</td>\n",
+       "      <td>6</td>\n",
+       "      <td>12000</td>\n",
+       "      <td>0.427343</td>\n",
+       "      <td>0.339</td>\n",
+       "      <td>0.348</td>\n",
+       "      <td>0.397</td>\n",
+       "      <td>0.439</td>\n",
+       "      <td>0.198</td>\n",
+       "      <td>0.316</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.382</td>\n",
+       "      <td>0.402</td>\n",
+       "      <td>0.535</td>\n",
+       "      <td>0.536</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.3705</td>\n",
+       "      <td>0.3795</td>\n",
+       "      <td>0.268891</td>\n",
+       "      <td>0.283246</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>132</th>\n",
+       "      <td>filtering-custom-short-line-ratio-0.67</td>\n",
+       "      <td>6</td>\n",
+       "      <td>13000</td>\n",
+       "      <td>0.429031</td>\n",
+       "      <td>0.338</td>\n",
+       "      <td>0.338</td>\n",
+       "      <td>0.398</td>\n",
+       "      <td>0.449</td>\n",
+       "      <td>0.194</td>\n",
+       "      <td>0.326</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.384</td>\n",
+       "      <td>0.406</td>\n",
+       "      <td>0.539</td>\n",
+       "      <td>0.534</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.3655</td>\n",
+       "      <td>0.3775</td>\n",
+       "      <td>0.271709</td>\n",
+       "      <td>0.282748</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>133</th>\n",
+       "      <td>filtering-custom-short-line-ratio-0.67</td>\n",
+       "      <td>6</td>\n",
+       "      <td>13500</td>\n",
+       "      <td>0.428488</td>\n",
+       "      <td>0.346</td>\n",
+       "      <td>0.340</td>\n",
+       "      <td>0.398</td>\n",
+       "      <td>0.447</td>\n",
+       "      <td>0.188</td>\n",
+       "      <td>0.332</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.382</td>\n",
+       "      <td>0.404</td>\n",
+       "      <td>0.527</td>\n",
+       "      <td>0.527</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.3720</td>\n",
+       "      <td>0.3730</td>\n",
+       "      <td>0.272315</td>\n",
+       "      <td>0.283901</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>134 rows × 22 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                    runname  seed  steps  agg_score  \\\n",
+       "0           filtering-baseline-2019-18-40gt     5      0   0.330953   \n",
+       "1           filtering-baseline-2019-18-40gt     5   1000   0.357474   \n",
+       "2           filtering-baseline-2019-18-40gt     5   2000   0.377436   \n",
+       "3           filtering-baseline-2019-18-40gt     5   3000   0.387994   \n",
+       "4           filtering-baseline-2019-18-40gt     5   4000   0.396110   \n",
+       "..                                      ...   ...    ...        ...   \n",
+       "129  filtering-custom-short-line-ratio-0.67     6  10000   0.422300   \n",
+       "130  filtering-custom-short-line-ratio-0.67     6  11000   0.425840   \n",
+       "131  filtering-custom-short-line-ratio-0.67     6  12000   0.427343   \n",
+       "132  filtering-custom-short-line-ratio-0.67     6  13000   0.429031   \n",
+       "133  filtering-custom-short-line-ratio-0.67     6  13500   0.428488   \n",
+       "\n",
+       "     commonsense_qa/acc  commonsense_qa/acc_norm  hellaswag/acc  \\\n",
+       "0                 0.186                    0.233          0.272   \n",
+       "1                 0.239                    0.271          0.297   \n",
+       "2                 0.280                    0.284          0.321   \n",
+       "3                 0.277                    0.291          0.339   \n",
+       "4                 0.299                    0.315          0.340   \n",
+       "..                  ...                      ...            ...   \n",
+       "129               0.333                    0.341          0.382   \n",
+       "130               0.345                    0.340          0.395   \n",
+       "131               0.339                    0.348          0.397   \n",
+       "132               0.338                    0.338          0.398   \n",
+       "133               0.346                    0.340          0.398   \n",
+       "\n",
+       "     hellaswag/acc_norm  openbookqa/acc  openbookqa/acc_norm  ...  siqa/acc  \\\n",
+       "0                 0.258           0.166                0.286  ...     0.367   \n",
+       "1                 0.287           0.146                0.260  ...     0.365   \n",
+       "2                 0.332           0.134                0.268  ...     0.368   \n",
+       "3                 0.359           0.132                0.280  ...     0.394   \n",
+       "4                 0.366           0.158                0.286  ...     0.376   \n",
+       "..                  ...             ...                  ...  ...       ...   \n",
+       "129               0.417           0.192                0.318  ...     0.389   \n",
+       "130               0.432           0.192                0.322  ...     0.379   \n",
+       "131               0.439           0.198                0.316  ...     0.382   \n",
+       "132               0.449           0.194                0.326  ...     0.384   \n",
+       "133               0.447           0.188                0.332  ...     0.382   \n",
+       "\n",
+       "     siqa/acc_norm  winogrande/acc  winogrande/acc_norm  sciq/acc  \\\n",
+       "0            0.362           0.516                0.497     0.210   \n",
+       "1            0.396           0.503                0.486     0.568   \n",
+       "2            0.399           0.519                0.502     0.686   \n",
+       "3            0.404           0.520                0.503     0.721   \n",
+       "4            0.399           0.515                0.500     0.739   \n",
+       "..             ...             ...                  ...       ...   \n",
+       "129          0.407           0.536                0.530       NaN   \n",
+       "130          0.405           0.527                0.531       NaN   \n",
+       "131          0.402           0.535                0.536       NaN   \n",
+       "132          0.406           0.539                0.534       NaN   \n",
+       "133          0.404           0.527                0.527       NaN   \n",
+       "\n",
+       "     sciq/acc_norm  arc/acc  arc/acc_norm  mmlu/acc  mmlu/acc_norm  \n",
+       "0            0.202   0.2190        0.2515  0.230285       0.250127  \n",
+       "1            0.502   0.2665        0.2855  0.242526       0.253291  \n",
+       "2            0.590   0.3030        0.3215  0.245745       0.260988  \n",
+       "3            0.622   0.3210        0.3385  0.250427       0.264451  \n",
+       "4            0.620   0.3320        0.3445  0.256134       0.270382  \n",
+       "..             ...      ...           ...       ...            ...  \n",
+       "129            NaN   0.3630        0.3700  0.266752       0.284400  \n",
+       "130            NaN   0.3680        0.3745  0.267998       0.282222  \n",
+       "131            NaN   0.3705        0.3795  0.268891       0.283246  \n",
+       "132            NaN   0.3655        0.3775  0.271709       0.282748  \n",
+       "133            NaN   0.3720        0.3730  0.272315       0.283901  \n",
+       "\n",
+       "[134 rows x 22 columns]"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "from matplotlib.figure import Figure\n",
+    "\n",
+    "df = pd.read_csv(\"../src_data/custom_filters.csv\")\n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "28e61084",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "runs_mapping = {\n",
+    "    \"filtering-baseline-2019-18-40gt\": \"Baseline\",\n",
+    "    \"filtering-custom-line-char-duplicated-v2-0.01\": \"Line duplicates filter\",\n",
+    "    \"filtering-custom-lines-punc-0.12\": \"Punctuation filter\",\n",
+    "    \"filtering-custom-short-line-ratio-0.67\": \"Short lines filter\",\n",
+    "    \"filtering-custom-punc0.12-short-lines0.67-line_char_dup0.1\": \"Filters combined\",\n",
+    "}\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "af28ebbd054cdc33",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-04T22:25:33.206952Z",
+     "start_time": "2024-05-04T22:25:33.205262Z"
+    },
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "\n",
+    "from collections import defaultdict\n",
+    "import json\n",
+    "import os\n",
+    "from matplotlib import pyplot as plt\n",
+    "import orjson\n",
+    "\n",
+    "metrics = ['agg_score', 'commonsense_qa/acc_norm', 'hellaswag/acc_norm', 'openbookqa/acc_norm', 'piqa/acc_norm',\n",
+    "                   'siqa/acc_norm', 'winogrande/acc_norm', 'arc/acc_norm', 'mmlu/acc_norm']\n",
+    "\n",
+    "def normalize_runname(runname):\n",
+    "    return runname.replace(\"/\", \"_\")\n",
+    "\n",
+    "grouped = (\n",
+    "    df.groupby([\"runname\", \"steps\"])\n",
+    "    .agg(\n",
+    "        {\n",
+    "            key: \"mean\" for key in metrics\n",
+    "        }\n",
+    "    )\n",
+    "    .reset_index()\n",
+    ")\n",
+    "\n",
+    "file_id=\"../assets/data/plots/custom_filters\"\n",
+    "files = {}\n",
+    "for metric in metrics:\n",
+    "    datas = {}\n",
+    "    for name, group in grouped.groupby(\"runname\"):\n",
+    "        group = group[[\"steps\", metric]].sort_values(by=\"steps\")\n",
+    "        group = group.set_index(\"steps\")\n",
+    "        rolling_avg = group\n",
+    "        # rolling_avg = group.rolling(window=5).mean()\n",
+    "        datas[name] = {\n",
+    "            \"x\": (rolling_avg.index * 2048 * 1024 * 1e-9).tolist(),\n",
+    "            \"y\": rolling_avg[metric].tolist(),\n",
+    "            \"label\": runs_mapping[name],\n",
+    "        }\n",
+    "    # Sort the datata based on the steps\n",
+    "    datas = {k: v for k, v in sorted(datas.items(), key=lambda x: -x[1][\"y\"][-1])}\n",
+    "    # Create a folder\n",
+    "    os.makedirs(f\"{file_id}\", exist_ok=True)\n",
+    "    with open(f\"{file_id}/{normalize_runname(metric)}.json\", \"w\") as f:\n",
+    "        json.dump({\n",
+    "            \"data\": datas,\n",
+    "            \"layout\": {\n",
+    "                \"title\": {\n",
+    "                    \"text\": \"Custom filters Performance\"\n",
+    "                },\n",
+    "            }\n",
+    "        }, f)\n",
+    "    files[metric] = {\"file\": f\"{normalize_runname(metric)}.json\"}\n",
+    "# Create index\n",
+    "with open(f\"{file_id}/index.json\", \"w\") as f:\n",
+    "    json.dump({\n",
+    "        \"files\": files,\n",
+    "        \"settings\": {\n",
+    "            \"defaultMetric\": \"agg_score\",\n",
+    "            \"slider\":{\"min\":0,\"max\":10,\"default\":3}\n",
+    "        }\n",
+    "    }, f)\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "80a14409",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/plot_dataset_ablations.ipynb

@@ -0,0 +1,533 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "138889b92720ce2e",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-14T09:06:04.487186Z",
+     "start_time": "2024-05-14T09:06:04.255111Z"
+    },
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>runname</th>\n",
+       "      <th>steps</th>\n",
+       "      <th>agg_score</th>\n",
+       "      <th>commonsense_qa/acc</th>\n",
+       "      <th>commonsense_qa/acc_norm</th>\n",
+       "      <th>hellaswag/acc</th>\n",
+       "      <th>hellaswag/acc_norm</th>\n",
+       "      <th>openbookqa/acc</th>\n",
+       "      <th>openbookqa/acc_norm</th>\n",
+       "      <th>piqa/acc</th>\n",
+       "      <th>...</th>\n",
+       "      <th>siqa/acc</th>\n",
+       "      <th>siqa/acc_norm</th>\n",
+       "      <th>winogrande/acc</th>\n",
+       "      <th>winogrande/acc_norm</th>\n",
+       "      <th>sciq/acc</th>\n",
+       "      <th>sciq/acc_norm</th>\n",
+       "      <th>arc/acc</th>\n",
+       "      <th>arc/acc_norm</th>\n",
+       "      <th>mmlu/acc</th>\n",
+       "      <th>mmlu/acc_norm</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>C4</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.330893</td>\n",
+       "      <td>0.186</td>\n",
+       "      <td>0.233</td>\n",
+       "      <td>0.272</td>\n",
+       "      <td>0.258</td>\n",
+       "      <td>0.166</td>\n",
+       "      <td>0.286</td>\n",
+       "      <td>0.542</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.367</td>\n",
+       "      <td>0.362</td>\n",
+       "      <td>0.516</td>\n",
+       "      <td>0.497</td>\n",
+       "      <td>0.208</td>\n",
+       "      <td>0.202</td>\n",
+       "      <td>0.2195</td>\n",
+       "      <td>0.2510</td>\n",
+       "      <td>0.230294</td>\n",
+       "      <td>0.250147</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>C4</td>\n",
+       "      <td>1000</td>\n",
+       "      <td>0.355112</td>\n",
+       "      <td>0.229</td>\n",
+       "      <td>0.260</td>\n",
+       "      <td>0.286</td>\n",
+       "      <td>0.288</td>\n",
+       "      <td>0.128</td>\n",
+       "      <td>0.250</td>\n",
+       "      <td>0.614</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.351</td>\n",
+       "      <td>0.404</td>\n",
+       "      <td>0.519</td>\n",
+       "      <td>0.476</td>\n",
+       "      <td>0.565</td>\n",
+       "      <td>0.518</td>\n",
+       "      <td>0.2680</td>\n",
+       "      <td>0.2935</td>\n",
+       "      <td>0.238951</td>\n",
+       "      <td>0.250399</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>C4</td>\n",
+       "      <td>2000</td>\n",
+       "      <td>0.378435</td>\n",
+       "      <td>0.268</td>\n",
+       "      <td>0.278</td>\n",
+       "      <td>0.312</td>\n",
+       "      <td>0.330</td>\n",
+       "      <td>0.122</td>\n",
+       "      <td>0.276</td>\n",
+       "      <td>0.646</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.375</td>\n",
+       "      <td>0.400</td>\n",
+       "      <td>0.509</td>\n",
+       "      <td>0.500</td>\n",
+       "      <td>0.676</td>\n",
+       "      <td>0.577</td>\n",
+       "      <td>0.3065</td>\n",
+       "      <td>0.3230</td>\n",
+       "      <td>0.247275</td>\n",
+       "      <td>0.255482</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>C4</td>\n",
+       "      <td>3000</td>\n",
+       "      <td>0.387795</td>\n",
+       "      <td>0.280</td>\n",
+       "      <td>0.295</td>\n",
+       "      <td>0.331</td>\n",
+       "      <td>0.380</td>\n",
+       "      <td>0.152</td>\n",
+       "      <td>0.274</td>\n",
+       "      <td>0.660</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.376</td>\n",
+       "      <td>0.387</td>\n",
+       "      <td>0.512</td>\n",
+       "      <td>0.496</td>\n",
+       "      <td>0.725</td>\n",
+       "      <td>0.621</td>\n",
+       "      <td>0.3175</td>\n",
+       "      <td>0.3340</td>\n",
+       "      <td>0.254534</td>\n",
+       "      <td>0.267363</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>C4</td>\n",
+       "      <td>4000</td>\n",
+       "      <td>0.399320</td>\n",
+       "      <td>0.296</td>\n",
+       "      <td>0.298</td>\n",
+       "      <td>0.351</td>\n",
+       "      <td>0.406</td>\n",
+       "      <td>0.168</td>\n",
+       "      <td>0.282</td>\n",
+       "      <td>0.676</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.382</td>\n",
+       "      <td>0.404</td>\n",
+       "      <td>0.522</td>\n",
+       "      <td>0.503</td>\n",
+       "      <td>0.723</td>\n",
+       "      <td>0.618</td>\n",
+       "      <td>0.3255</td>\n",
+       "      <td>0.3470</td>\n",
+       "      <td>0.254762</td>\n",
+       "      <td>0.263563</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1171</th>\n",
+       "      <td>The Pile</td>\n",
+       "      <td>163000</td>\n",
+       "      <td>0.463789</td>\n",
+       "      <td>0.379</td>\n",
+       "      <td>0.349</td>\n",
+       "      <td>0.441</td>\n",
+       "      <td>0.555</td>\n",
+       "      <td>0.240</td>\n",
+       "      <td>0.366</td>\n",
+       "      <td>0.701</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.405</td>\n",
+       "      <td>0.388</td>\n",
+       "      <td>0.585</td>\n",
+       "      <td>0.560</td>\n",
+       "      <td>0.875</td>\n",
+       "      <td>0.820</td>\n",
+       "      <td>0.4475</td>\n",
+       "      <td>0.4450</td>\n",
+       "      <td>0.299378</td>\n",
+       "      <td>0.326313</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1172</th>\n",
+       "      <td>The Pile</td>\n",
+       "      <td>164000</td>\n",
+       "      <td>0.462758</td>\n",
+       "      <td>0.369</td>\n",
+       "      <td>0.344</td>\n",
+       "      <td>0.438</td>\n",
+       "      <td>0.552</td>\n",
+       "      <td>0.248</td>\n",
+       "      <td>0.348</td>\n",
+       "      <td>0.708</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.395</td>\n",
+       "      <td>0.401</td>\n",
+       "      <td>0.577</td>\n",
+       "      <td>0.567</td>\n",
+       "      <td>0.874</td>\n",
+       "      <td>0.806</td>\n",
+       "      <td>0.4465</td>\n",
+       "      <td>0.4355</td>\n",
+       "      <td>0.302083</td>\n",
+       "      <td>0.331563</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1173</th>\n",
+       "      <td>The Pile</td>\n",
+       "      <td>165000</td>\n",
+       "      <td>0.465026</td>\n",
+       "      <td>0.383</td>\n",
+       "      <td>0.350</td>\n",
+       "      <td>0.438</td>\n",
+       "      <td>0.553</td>\n",
+       "      <td>0.234</td>\n",
+       "      <td>0.352</td>\n",
+       "      <td>0.707</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.400</td>\n",
+       "      <td>0.401</td>\n",
+       "      <td>0.569</td>\n",
+       "      <td>0.556</td>\n",
+       "      <td>0.874</td>\n",
+       "      <td>0.811</td>\n",
+       "      <td>0.4460</td>\n",
+       "      <td>0.4455</td>\n",
+       "      <td>0.305193</td>\n",
+       "      <td>0.331708</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1174</th>\n",
+       "      <td>The Pile</td>\n",
+       "      <td>166000</td>\n",
+       "      <td>0.462349</td>\n",
+       "      <td>0.377</td>\n",
+       "      <td>0.346</td>\n",
+       "      <td>0.440</td>\n",
+       "      <td>0.557</td>\n",
+       "      <td>0.228</td>\n",
+       "      <td>0.346</td>\n",
+       "      <td>0.711</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.398</td>\n",
+       "      <td>0.398</td>\n",
+       "      <td>0.572</td>\n",
+       "      <td>0.558</td>\n",
+       "      <td>0.877</td>\n",
+       "      <td>0.811</td>\n",
+       "      <td>0.4525</td>\n",
+       "      <td>0.4385</td>\n",
+       "      <td>0.301952</td>\n",
+       "      <td>0.331295</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1175</th>\n",
+       "      <td>The Pile</td>\n",
+       "      <td>167000</td>\n",
+       "      <td>0.464539</td>\n",
+       "      <td>0.386</td>\n",
+       "      <td>0.354</td>\n",
+       "      <td>0.434</td>\n",
+       "      <td>0.557</td>\n",
+       "      <td>0.232</td>\n",
+       "      <td>0.356</td>\n",
+       "      <td>0.706</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.402</td>\n",
+       "      <td>0.402</td>\n",
+       "      <td>0.573</td>\n",
+       "      <td>0.559</td>\n",
+       "      <td>0.867</td>\n",
+       "      <td>0.802</td>\n",
+       "      <td>0.4475</td>\n",
+       "      <td>0.4375</td>\n",
+       "      <td>0.301934</td>\n",
+       "      <td>0.330810</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>1176 rows × 21 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "       runname   steps  agg_score  commonsense_qa/acc  \\\n",
+       "0           C4       0   0.330893               0.186   \n",
+       "1           C4    1000   0.355112               0.229   \n",
+       "2           C4    2000   0.378435               0.268   \n",
+       "3           C4    3000   0.387795               0.280   \n",
+       "4           C4    4000   0.399320               0.296   \n",
+       "...        ...     ...        ...                 ...   \n",
+       "1171  The Pile  163000   0.463789               0.379   \n",
+       "1172  The Pile  164000   0.462758               0.369   \n",
+       "1173  The Pile  165000   0.465026               0.383   \n",
+       "1174  The Pile  166000   0.462349               0.377   \n",
+       "1175  The Pile  167000   0.464539               0.386   \n",
+       "\n",
+       "      commonsense_qa/acc_norm  hellaswag/acc  hellaswag/acc_norm  \\\n",
+       "0                       0.233          0.272               0.258   \n",
+       "1                       0.260          0.286               0.288   \n",
+       "2                       0.278          0.312               0.330   \n",
+       "3                       0.295          0.331               0.380   \n",
+       "4                       0.298          0.351               0.406   \n",
+       "...                       ...            ...                 ...   \n",
+       "1171                    0.349          0.441               0.555   \n",
+       "1172                    0.344          0.438               0.552   \n",
+       "1173                    0.350          0.438               0.553   \n",
+       "1174                    0.346          0.440               0.557   \n",
+       "1175                    0.354          0.434               0.557   \n",
+       "\n",
+       "      openbookqa/acc  openbookqa/acc_norm  piqa/acc  ...  siqa/acc  \\\n",
+       "0              0.166                0.286     0.542  ...     0.367   \n",
+       "1              0.128                0.250     0.614  ...     0.351   \n",
+       "2              0.122                0.276     0.646  ...     0.375   \n",
+       "3              0.152                0.274     0.660  ...     0.376   \n",
+       "4              0.168                0.282     0.676  ...     0.382   \n",
+       "...              ...                  ...       ...  ...       ...   \n",
+       "1171           0.240                0.366     0.701  ...     0.405   \n",
+       "1172           0.248                0.348     0.708  ...     0.395   \n",
+       "1173           0.234                0.352     0.707  ...     0.400   \n",
+       "1174           0.228                0.346     0.711  ...     0.398   \n",
+       "1175           0.232                0.356     0.706  ...     0.402   \n",
+       "\n",
+       "      siqa/acc_norm  winogrande/acc  winogrande/acc_norm  sciq/acc  \\\n",
+       "0             0.362           0.516                0.497     0.208   \n",
+       "1             0.404           0.519                0.476     0.565   \n",
+       "2             0.400           0.509                0.500     0.676   \n",
+       "3             0.387           0.512                0.496     0.725   \n",
+       "4             0.404           0.522                0.503     0.723   \n",
+       "...             ...             ...                  ...       ...   \n",
+       "1171          0.388           0.585                0.560     0.875   \n",
+       "1172          0.401           0.577                0.567     0.874   \n",
+       "1173          0.401           0.569                0.556     0.874   \n",
+       "1174          0.398           0.572                0.558     0.877   \n",
+       "1175          0.402           0.573                0.559     0.867   \n",
+       "\n",
+       "      sciq/acc_norm  arc/acc  arc/acc_norm  mmlu/acc  mmlu/acc_norm  \n",
+       "0             0.202   0.2195        0.2510  0.230294       0.250147  \n",
+       "1             0.518   0.2680        0.2935  0.238951       0.250399  \n",
+       "2             0.577   0.3065        0.3230  0.247275       0.255482  \n",
+       "3             0.621   0.3175        0.3340  0.254534       0.267363  \n",
+       "4             0.618   0.3255        0.3470  0.254762       0.263563  \n",
+       "...             ...      ...           ...       ...            ...  \n",
+       "1171          0.820   0.4475        0.4450  0.299378       0.326313  \n",
+       "1172          0.806   0.4465        0.4355  0.302083       0.331563  \n",
+       "1173          0.811   0.4460        0.4455  0.305193       0.331708  \n",
+       "1174          0.811   0.4525        0.4385  0.301952       0.331295  \n",
+       "1175          0.802   0.4475        0.4375  0.301934       0.330810  \n",
+       "\n",
+       "[1176 rows x 21 columns]"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "from matplotlib.figure import Figure\n",
+    "\n",
+    "df = pd.read_csv(\"../src_data/eval_results.csv\")\n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "b610f43caefdf01",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-14T09:06:04.563945Z",
+     "start_time": "2024-05-14T09:06:04.562142Z"
+    },
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "initial_id",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-14T09:06:37.927921Z",
+     "start_time": "2024-05-14T09:06:37.588025Z"
+    },
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "import os\n",
+    "from matplotlib import pyplot as plt\n",
+    "metrics = ['agg_score', 'commonsense_qa/acc_norm', 'hellaswag/acc_norm', 'openbookqa/acc_norm', 'piqa/acc_norm',\n",
+    "                   'siqa/acc_norm', 'winogrande/acc_norm', 'arc/acc_norm', 'mmlu/acc_norm']\n",
+    "\n",
+    "def normalize_runname(runname):\n",
+    "    return runname.replace(\"/\", \"_\")\n",
+    "\n",
+    "grouped = (\n",
+    "    df.groupby([\"runname\", \"steps\"])\n",
+    "    .agg(\n",
+    "        {\n",
+    "            key: \"mean\" for key in metrics\n",
+    "        }\n",
+    "    )\n",
+    "    .reset_index()\n",
+    ")\n",
+    "\n",
+    "file_id=\"../assets/data/plots/dataset_ablations\"\n",
+    "files = {}\n",
+    "for metric in metrics:\n",
+    "    datas = {}\n",
+    "    for name, group in grouped.groupby(\"runname\"):\n",
+    "        group = group[[\"steps\", metric]].sort_values(by=\"steps\")\n",
+    "        group = group.set_index(\"steps\")\n",
+    "        rolling_avg = group\n",
+    "        # rolling_avg = group.rolling(window=5).mean()\n",
+    "        datas[name] = {\n",
+    "            \"x\": (rolling_avg.index * 2048 * 1024 * 1e-9).tolist(),\n",
+    "            \"y\": rolling_avg[metric].tolist(),\n",
+    "            \"label\": name,\n",
+    "        }\n",
+    "    # Sort the datata based on the steps\n",
+    "    datas = {k: v for k, v in sorted(datas.items(), key=lambda x: -x[1][\"y\"][-1])}\n",
+    "    # Create a folder\n",
+    "    os.makedirs(f\"{file_id}\", exist_ok=True)\n",
+    "    with open(f\"{file_id}/{normalize_runname(metric)}.json\", \"w\") as f:\n",
+    "        json.dump({\n",
+    "            \"data\": datas,\n",
+    "            \"layout\": {\n",
+    "                \"title\": {\n",
+    "                    \"text\": \"Dataset ablations\"\n",
+    "                },\n",
+    "            }\n",
+    "        }, f)\n",
+    "    files[metric] = {\"file\": f\"{normalize_runname(metric)}.json\"}\n",
+    "# Create index\n",
+    "with open(f\"{file_id}/index.json\", \"w\") as f:\n",
+    "    json.dump({\n",
+    "        \"files\": files,\n",
+    "        \"settings\": {\n",
+    "            \"defaultMetric\": \"agg_score\",\n",
+    "            \"slider\":{\"min\":0,\"max\":30,\"default\":5}\n",
+    "        }\n",
+    "    }, f)\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "af28ebbd054cdc33",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-04T22:25:33.206952Z",
+     "start_time": "2024-05-04T22:25:33.205262Z"
+    },
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/plot_dedup_all_dumps_bad.ipynb

@@ -0,0 +1,569 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "138889b92720ce2e",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-04-30T15:07:36.238754Z",
+     "start_time": "2024-04-30T15:07:35.974657Z"
+    },
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>runname</th>\n",
+       "      <th>seed</th>\n",
+       "      <th>steps</th>\n",
+       "      <th>agg_score</th>\n",
+       "      <th>commonsense_qa/acc</th>\n",
+       "      <th>commonsense_qa/acc_norm</th>\n",
+       "      <th>hellaswag/acc</th>\n",
+       "      <th>hellaswag/acc_norm</th>\n",
+       "      <th>openbookqa/acc</th>\n",
+       "      <th>openbookqa/acc_norm</th>\n",
+       "      <th>...</th>\n",
+       "      <th>siqa/acc</th>\n",
+       "      <th>siqa/acc_norm</th>\n",
+       "      <th>winogrande/acc</th>\n",
+       "      <th>winogrande/acc_norm</th>\n",
+       "      <th>sciq/acc</th>\n",
+       "      <th>sciq/acc_norm</th>\n",
+       "      <th>arc/acc</th>\n",
+       "      <th>arc/acc_norm</th>\n",
+       "      <th>mmlu/acc</th>\n",
+       "      <th>mmlu/acc_norm</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>big-run-sampled_full_filtered_no_dedup</td>\n",
+       "      <td>6</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.330893</td>\n",
+       "      <td>0.186</td>\n",
+       "      <td>0.233</td>\n",
+       "      <td>0.272</td>\n",
+       "      <td>0.258</td>\n",
+       "      <td>0.166</td>\n",
+       "      <td>0.286</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.367</td>\n",
+       "      <td>0.362</td>\n",
+       "      <td>0.516</td>\n",
+       "      <td>0.497</td>\n",
+       "      <td>0.209</td>\n",
+       "      <td>0.202</td>\n",
+       "      <td>0.2195</td>\n",
+       "      <td>0.2510</td>\n",
+       "      <td>0.230294</td>\n",
+       "      <td>0.250147</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>big-run-sampled_full_filtered_no_dedup</td>\n",
+       "      <td>6</td>\n",
+       "      <td>1000</td>\n",
+       "      <td>0.360520</td>\n",
+       "      <td>0.254</td>\n",
+       "      <td>0.260</td>\n",
+       "      <td>0.290</td>\n",
+       "      <td>0.281</td>\n",
+       "      <td>0.138</td>\n",
+       "      <td>0.256</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.362</td>\n",
+       "      <td>0.400</td>\n",
+       "      <td>0.517</td>\n",
+       "      <td>0.524</td>\n",
+       "      <td>0.573</td>\n",
+       "      <td>0.515</td>\n",
+       "      <td>0.2675</td>\n",
+       "      <td>0.2895</td>\n",
+       "      <td>0.239489</td>\n",
+       "      <td>0.251660</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>big-run-sampled_full_filtered_no_dedup</td>\n",
+       "      <td>6</td>\n",
+       "      <td>2000</td>\n",
+       "      <td>0.373315</td>\n",
+       "      <td>0.285</td>\n",
+       "      <td>0.278</td>\n",
+       "      <td>0.315</td>\n",
+       "      <td>0.323</td>\n",
+       "      <td>0.138</td>\n",
+       "      <td>0.272</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.365</td>\n",
+       "      <td>0.395</td>\n",
+       "      <td>0.509</td>\n",
+       "      <td>0.490</td>\n",
+       "      <td>0.677</td>\n",
+       "      <td>0.596</td>\n",
+       "      <td>0.3075</td>\n",
+       "      <td>0.3235</td>\n",
+       "      <td>0.250318</td>\n",
+       "      <td>0.261019</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>big-run-sampled_full_filtered_no_dedup</td>\n",
+       "      <td>6</td>\n",
+       "      <td>3000</td>\n",
+       "      <td>0.388201</td>\n",
+       "      <td>0.294</td>\n",
+       "      <td>0.291</td>\n",
+       "      <td>0.327</td>\n",
+       "      <td>0.341</td>\n",
+       "      <td>0.152</td>\n",
+       "      <td>0.298</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.371</td>\n",
+       "      <td>0.396</td>\n",
+       "      <td>0.512</td>\n",
+       "      <td>0.504</td>\n",
+       "      <td>0.712</td>\n",
+       "      <td>0.621</td>\n",
+       "      <td>0.3220</td>\n",
+       "      <td>0.3390</td>\n",
+       "      <td>0.255646</td>\n",
+       "      <td>0.266605</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>big-run-sampled_full_filtered_no_dedup</td>\n",
+       "      <td>6</td>\n",
+       "      <td>4000</td>\n",
+       "      <td>0.393412</td>\n",
+       "      <td>0.306</td>\n",
+       "      <td>0.307</td>\n",
+       "      <td>0.337</td>\n",
+       "      <td>0.360</td>\n",
+       "      <td>0.172</td>\n",
+       "      <td>0.284</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.380</td>\n",
+       "      <td>0.402</td>\n",
+       "      <td>0.522</td>\n",
+       "      <td>0.510</td>\n",
+       "      <td>0.729</td>\n",
+       "      <td>0.612</td>\n",
+       "      <td>0.3100</td>\n",
+       "      <td>0.3385</td>\n",
+       "      <td>0.253048</td>\n",
+       "      <td>0.266798</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>501</th>\n",
+       "      <td>big-run-fineweb-cross-dedup-fixed</td>\n",
+       "      <td>6</td>\n",
+       "      <td>163000</td>\n",
+       "      <td>0.466306</td>\n",
+       "      <td>0.391</td>\n",
+       "      <td>0.371</td>\n",
+       "      <td>0.459</td>\n",
+       "      <td>0.547</td>\n",
+       "      <td>0.210</td>\n",
+       "      <td>0.344</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.401</td>\n",
+       "      <td>0.388</td>\n",
+       "      <td>0.564</td>\n",
+       "      <td>0.562</td>\n",
+       "      <td>0.884</td>\n",
+       "      <td>0.807</td>\n",
+       "      <td>0.4535</td>\n",
+       "      <td>0.4450</td>\n",
+       "      <td>0.300475</td>\n",
+       "      <td>0.320448</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>502</th>\n",
+       "      <td>big-run-fineweb-cross-dedup-fixed</td>\n",
+       "      <td>6</td>\n",
+       "      <td>164000</td>\n",
+       "      <td>0.468313</td>\n",
+       "      <td>0.395</td>\n",
+       "      <td>0.374</td>\n",
+       "      <td>0.459</td>\n",
+       "      <td>0.548</td>\n",
+       "      <td>0.208</td>\n",
+       "      <td>0.350</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.402</td>\n",
+       "      <td>0.395</td>\n",
+       "      <td>0.559</td>\n",
+       "      <td>0.561</td>\n",
+       "      <td>0.876</td>\n",
+       "      <td>0.795</td>\n",
+       "      <td>0.4540</td>\n",
+       "      <td>0.4445</td>\n",
+       "      <td>0.299279</td>\n",
+       "      <td>0.321007</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>503</th>\n",
+       "      <td>big-run-fineweb-cross-dedup-fixed</td>\n",
+       "      <td>6</td>\n",
+       "      <td>165000</td>\n",
+       "      <td>0.468639</td>\n",
+       "      <td>0.397</td>\n",
+       "      <td>0.374</td>\n",
+       "      <td>0.450</td>\n",
+       "      <td>0.548</td>\n",
+       "      <td>0.208</td>\n",
+       "      <td>0.358</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.400</td>\n",
+       "      <td>0.391</td>\n",
+       "      <td>0.552</td>\n",
+       "      <td>0.556</td>\n",
+       "      <td>0.876</td>\n",
+       "      <td>0.787</td>\n",
+       "      <td>0.4490</td>\n",
+       "      <td>0.4420</td>\n",
+       "      <td>0.298460</td>\n",
+       "      <td>0.319108</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>504</th>\n",
+       "      <td>big-run-fineweb-cross-dedup-fixed</td>\n",
+       "      <td>6</td>\n",
+       "      <td>166000</td>\n",
+       "      <td>0.465767</td>\n",
+       "      <td>0.412</td>\n",
+       "      <td>0.375</td>\n",
+       "      <td>0.458</td>\n",
+       "      <td>0.552</td>\n",
+       "      <td>0.214</td>\n",
+       "      <td>0.348</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.403</td>\n",
+       "      <td>0.398</td>\n",
+       "      <td>0.551</td>\n",
+       "      <td>0.553</td>\n",
+       "      <td>0.877</td>\n",
+       "      <td>0.802</td>\n",
+       "      <td>0.4465</td>\n",
+       "      <td>0.4345</td>\n",
+       "      <td>0.298333</td>\n",
+       "      <td>0.318637</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>505</th>\n",
+       "      <td>big-run-fineweb-cross-dedup-fixed</td>\n",
+       "      <td>6</td>\n",
+       "      <td>167000</td>\n",
+       "      <td>0.469262</td>\n",
+       "      <td>0.399</td>\n",
+       "      <td>0.377</td>\n",
+       "      <td>0.459</td>\n",
+       "      <td>0.550</td>\n",
+       "      <td>0.220</td>\n",
+       "      <td>0.348</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.406</td>\n",
+       "      <td>0.401</td>\n",
+       "      <td>0.564</td>\n",
+       "      <td>0.560</td>\n",
+       "      <td>0.882</td>\n",
+       "      <td>0.798</td>\n",
+       "      <td>0.4480</td>\n",
+       "      <td>0.4405</td>\n",
+       "      <td>0.297617</td>\n",
+       "      <td>0.319592</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>506 rows × 22 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                    runname  seed   steps  agg_score  \\\n",
+       "0    big-run-sampled_full_filtered_no_dedup     6       0   0.330893   \n",
+       "1    big-run-sampled_full_filtered_no_dedup     6    1000   0.360520   \n",
+       "2    big-run-sampled_full_filtered_no_dedup     6    2000   0.373315   \n",
+       "3    big-run-sampled_full_filtered_no_dedup     6    3000   0.388201   \n",
+       "4    big-run-sampled_full_filtered_no_dedup     6    4000   0.393412   \n",
+       "..                                      ...   ...     ...        ...   \n",
+       "501       big-run-fineweb-cross-dedup-fixed     6  163000   0.466306   \n",
+       "502       big-run-fineweb-cross-dedup-fixed     6  164000   0.468313   \n",
+       "503       big-run-fineweb-cross-dedup-fixed     6  165000   0.468639   \n",
+       "504       big-run-fineweb-cross-dedup-fixed     6  166000   0.465767   \n",
+       "505       big-run-fineweb-cross-dedup-fixed     6  167000   0.469262   \n",
+       "\n",
+       "     commonsense_qa/acc  commonsense_qa/acc_norm  hellaswag/acc  \\\n",
+       "0                 0.186                    0.233          0.272   \n",
+       "1                 0.254                    0.260          0.290   \n",
+       "2                 0.285                    0.278          0.315   \n",
+       "3                 0.294                    0.291          0.327   \n",
+       "4                 0.306                    0.307          0.337   \n",
+       "..                  ...                      ...            ...   \n",
+       "501               0.391                    0.371          0.459   \n",
+       "502               0.395                    0.374          0.459   \n",
+       "503               0.397                    0.374          0.450   \n",
+       "504               0.412                    0.375          0.458   \n",
+       "505               0.399                    0.377          0.459   \n",
+       "\n",
+       "     hellaswag/acc_norm  openbookqa/acc  openbookqa/acc_norm  ...  siqa/acc  \\\n",
+       "0                 0.258           0.166                0.286  ...     0.367   \n",
+       "1                 0.281           0.138                0.256  ...     0.362   \n",
+       "2                 0.323           0.138                0.272  ...     0.365   \n",
+       "3                 0.341           0.152                0.298  ...     0.371   \n",
+       "4                 0.360           0.172                0.284  ...     0.380   \n",
+       "..                  ...             ...                  ...  ...       ...   \n",
+       "501               0.547           0.210                0.344  ...     0.401   \n",
+       "502               0.548           0.208                0.350  ...     0.402   \n",
+       "503               0.548           0.208                0.358  ...     0.400   \n",
+       "504               0.552           0.214                0.348  ...     0.403   \n",
+       "505               0.550           0.220                0.348  ...     0.406   \n",
+       "\n",
+       "     siqa/acc_norm  winogrande/acc  winogrande/acc_norm  sciq/acc  \\\n",
+       "0            0.362           0.516                0.497     0.209   \n",
+       "1            0.400           0.517                0.524     0.573   \n",
+       "2            0.395           0.509                0.490     0.677   \n",
+       "3            0.396           0.512                0.504     0.712   \n",
+       "4            0.402           0.522                0.510     0.729   \n",
+       "..             ...             ...                  ...       ...   \n",
+       "501          0.388           0.564                0.562     0.884   \n",
+       "502          0.395           0.559                0.561     0.876   \n",
+       "503          0.391           0.552                0.556     0.876   \n",
+       "504          0.398           0.551                0.553     0.877   \n",
+       "505          0.401           0.564                0.560     0.882   \n",
+       "\n",
+       "     sciq/acc_norm  arc/acc  arc/acc_norm  mmlu/acc  mmlu/acc_norm  \n",
+       "0            0.202   0.2195        0.2510  0.230294       0.250147  \n",
+       "1            0.515   0.2675        0.2895  0.239489       0.251660  \n",
+       "2            0.596   0.3075        0.3235  0.250318       0.261019  \n",
+       "3            0.621   0.3220        0.3390  0.255646       0.266605  \n",
+       "4            0.612   0.3100        0.3385  0.253048       0.266798  \n",
+       "..             ...      ...           ...       ...            ...  \n",
+       "501          0.807   0.4535        0.4450  0.300475       0.320448  \n",
+       "502          0.795   0.4540        0.4445  0.299279       0.321007  \n",
+       "503          0.787   0.4490        0.4420  0.298460       0.319108  \n",
+       "504          0.802   0.4465        0.4345  0.298333       0.318637  \n",
+       "505          0.798   0.4480        0.4405  0.297617       0.319592  \n",
+       "\n",
+       "[506 rows x 22 columns]"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "from matplotlib.figure import Figure\n",
+    "\n",
+    "df = pd.read_csv(\"../src_data/cross_dedup_refinedweb_filtered.csv\")\n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "b610f43caefdf01",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-04-30T15:07:36.242016Z",
+     "start_time": "2024-04-30T15:07:36.239657Z"
+    },
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "runs_mapping = {\n",
+    "    \"big-run-refinedweb\": \"RefinedWeb\",\n",
+    "    \"big-run-fineweb-cross-dedup-fixed\": \"FineWeb full MinHash\",\n",
+    "    \"big-run-sampled_full_filtered_no_dedup\": \"FineWeb filtered only\"\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "initial_id",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-04-30T15:07:36.360665Z",
+     "start_time": "2024-04-30T15:07:36.242724Z"
+    },
+    "collapsed": true
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
+     ]
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkIAAAHHCAYAAABTMjf2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAA9hAAAPYQGoP6dpAABML0lEQVR4nO3dd1QU198G8Gcpu4BUBQERQbGB/UUlWIIFxRKNxgiWCPaoWImJGgsaC8ae2DXGFhM1dmMXNUbFXmPvGgUUFVBQEPa+f3iYnyuLssiywDyfc/Yc9u6dme/s3fIwbRVCCAEiIiIiGTIydAFEREREhsIgRERERLLFIERERESyxSBEREREssUgRERERLLFIERERESyxSBEREREssUgRERERLLFIERERESyxSBUwCxbtgwKhQJ37twxdCkEoEGDBmjQoIF0/86dO1AoFFi2bFmO5ifX8e3atSvc3d0NXUa+dODAASgUChw4cMDQpdBbrl+/jqZNm8LGxgYKhQKbNm0ydEmUQwxCHyHjSyvjZmZmhhIlSiAgIAA///wznj9/bugSiSifSU5OxtixYxlsCriQkBBcuHABEydOxMqVK1GzZk1Dl0Q5ZGLoAgqDH374AaVLl8br168RExODAwcOYPDgwZgxYwa2bNmCqlWrGrpEIsonkpOTMW7cOADQ2JqYlU8//RQvX76EUqnUc2WUXS9fvkRUVBRGjhyJ/v37G7oc+kgMQrmgefPmGv8NjBgxAvv27cNnn32G1q1b4/LlyzA3NzdghQVDUlISihQpYugyiPIVIyMjmJmZGboMiVqtRmpqar6qKa+8evUKSqUSjx8/BgDY2trm2rz5+Wc43DWmJ40aNcLo0aNx9+5d/PbbbxqPXblyBV9++SWKFi0KMzMz1KxZE1u2bMk0j4sXL6JRo0YwNzdHyZIlMWHCBKjV6kz9FAoFxo4dm6nd3d0dXbt2le5n7Mo7ePAgvv76axQrVgzW1tYIDg7Gs2fPPrhO58+fR9euXVGmTBmYmZnByckJ3bt3x5MnTzL1ffDgAXr06IESJUpApVKhdOnS6Nu3L1JTUzVq+fvvv9GvXz8UL14cJUuWlKafN28eKlWqBJVKhRIlSiA0NBTx8fEay7h+/TratWsHJycnmJmZoWTJkujQoQMSEhKkPnv27EG9evVga2sLS0tLVKhQAd9///0H13Xp0qVo1KgRihcvDpVKBS8vL8yfP/+D0+lCX+N76NAhDBw4EA4ODrC1tcXXX3+N1NRUxMfHIzg4GHZ2drCzs8N3330HIYQ0fcbxTdOmTcPMmTPh5uYGc3Nz+Pn54d9//9VYdkxMDLp164aSJUtCpVLB2dkZn3/+ebaObdq0aRMqV64MMzMzVK5cGRs3btTaLykpCd988w1cXV2hUqlQoUIFTJs2TaPmDL/99hu8vb1hbm6OokWLokOHDrh//75Gn+y8XrRp0KABKleujEuXLqFhw4awsLCAi4sLpkyZkqnvo0eP0KNHDzg6OsLMzAzVqlXD8uXLpcfv3LkDBwcHAMC4ceOk3eraxjeDtmOEMmo6f/48/Pz8YGFhgbJly2LdunUAgL///hs+Pj4wNzdHhQoVsHfvXo15jh07FgqFAleuXEFgYCCsra1RrFgxDBo0CK9evdLoq1Ao0L9/f6xatUp6T+7cuRMAcObMGTRv3hzW1tawtLRE48aNcfToUWnakydPQqFQaDwHGXbt2gWFQoG//vpLanvw4AG6d+8OR0dHqFQqVKpUCb/++muWz01WdVaoUAFmZmbw9vbGwYMHM/XNznIynvfVq1dj1KhRcHFxgYWFBcLCwuDm5gYA+Pbbb6FQKDSOb/vQcwK8//PvY8f27t276NevHypUqABzc3MUK1YM7du3z/TezKjh8OHDCAsLg4ODA4oUKYK2bdtKQe9tO3bsgJ+fH6ysrGBtbY1atWrh999/1+hz7NgxNGvWDDY2NrCwsICfnx8OHz78nlHLH7hFSI+6dOmC77//Hrt370avXr0AvPnyq1u3LlxcXDB8+HAUKVIEa9euRZs2bbB+/Xq0bdsWwJsvmoYNGyItLU3qt2jRolzZstS/f3/Y2tpi7NixuHr1KubPn4+7d+9Kb/ys7NmzB7du3UK3bt3g5OSEixcvYtGiRbh48SKOHj0qTfvw4UPUrl0b8fHx6N27NypWrIgHDx5g3bp1SE5O1tjE369fPzg4OGDMmDFISkoC8OZDety4cfD390ffvn2lGk+cOIHDhw/D1NQUqampCAgIQEpKCgYMGAAnJyc8ePAAf/31F+Lj42FjY4OLFy/is88+Q9WqVfHDDz9ApVLhxo0b2Xpjzp8/H5UqVULr1q1hYmKCrVu3ol+/flCr1QgNDf3IEdDv+GY8H+PGjcPRo0exaNEi2Nra4siRIyhVqhQmTZqE7du3Y+rUqahcuTKCg4M1pl+xYgWeP3+O0NBQvHr1Cj/99BMaNWqECxcuwNHREQDQrl07XLx4EQMGDIC7uzsePXqEPXv24N69e+896Hn37t1o164dvLy8EBERgSdPnkiB6m1CCLRu3Rr79+9Hjx49UL16dezatQvffvstHjx4gJkzZ0p9J06ciNGjRyMwMBA9e/bE48ePMXv2bHz66ac4c+YMbG1ts/V6eZ9nz56hWbNm+OKLLxAYGIh169Zh2LBhqFKlCpo3bw7gze6SBg0a4MaNG+jfvz9Kly6NP//8E127dkV8fDwGDRoEBwcHzJ8/H3379kXbtm3xxRdfAECOdp8/e/YMn332GTp06ID27dtj/vz56NChA1atWoXBgwejT58+6NSpE6ZOnYovv/wS9+/fh5WVlcY8AgMD4e7ujoiICBw9ehQ///wznj17hhUrVmj027dvH9auXYv+/fvD3t4e7u7uuHjxIurXrw9ra2t89913MDU1xcKFC9GgQQPpy7pmzZooU6YM1q5di5CQEI15rlmzBnZ2dggICAAAxMbG4pNPPpECjYODA3bs2IEePXogMTERgwcP/uBz8vfff2PNmjUYOHAgVCoV5s2bh2bNmuH48eOoXLlyjpYzfvx4KJVKDB06FCkpKWjRogXc3d0xZMgQdOzYES1atIClpSUAZOs5eZu2z7+PHdsTJ07gyJEj6NChA0qWLIk7d+5g/vz5aNCgAS5dugQLCwuNGgYMGAA7OzuEh4fjzp07mDVrFvr37481a9ZIfZYtW4bu3bujUqVKGDFiBGxtbXHmzBns3LkTnTp1kl4jzZs3h7e3N8LDw2FkZCT9Q/nPP/+gdu3aHxw/gxGUY0uXLhUAxIkTJ7LsY2NjI2rUqCHdb9y4sahSpYp49eqV1KZWq0WdOnVEuXLlpLbBgwcLAOLYsWNS26NHj4SNjY0AIG7fvi21AxDh4eGZlu3m5iZCQkIy1evt7S1SU1Ol9ilTpggAYvPmze9d3+Tk5Extf/zxhwAgDh48KLUFBwcLIyMjrc+LWq3WqKVevXoiLS1NYx2VSqVo2rSpSE9Pl9rnzJkjAIhff/1VCCHEmTNnBADx559/ZlnvzJkzBQDx+PHj965Xdtc1ICBAlClTRqPNz89P+Pn5Sfdv374tAIilS5e+d/76HN+AgADpeRZCCF9fX6FQKESfPn2ktrS0NFGyZEmttZubm4v//vtPaj927JgAIIYMGSKEEOLZs2cCgJg6dep711Gb6tWrC2dnZxEfHy+17d69WwAQbm5uUtumTZsEADFhwgSN6b/88kuhUCjEjRs3hBBC3LlzRxgbG4uJEydq9Ltw4YIwMTGR2rPzesmKn5+fACBWrFghtaWkpAgnJyfRrl07qW3WrFkCgPjtt9+kttTUVOHr6yssLS1FYmKiEEKIx48fZzmm2uzfv18AEPv3789U0++//y61XblyRQAQRkZG4ujRo1L7rl27Mr0mw8PDBQDRunVrjWX169dPABDnzp2T2jLmefHiRY2+bdq0EUqlUty8eVNqe/jwobCyshKffvqp1DZixAhhamoqnj59KrWlpKQIW1tb0b17d6mtR48ewtnZWcTFxWksp0OHDsLGxkbre/JtAAQAcfLkSant7t27wszMTLRt21bn5WQ872XKlMm07Iz3yrvvgew+J1l9/gnx8WOr7XmKiorK9BrOqMHf31/j82LIkCHC2NhYeo/Gx8cLKysr4ePjI16+fKkx34zp1Gq1KFeuXKbPnuTkZFG6dGnRpEmTTDXlJ9w1pmeWlpbS2WNPnz7Fvn37EBgYiOfPnyMuLg5xcXF48uQJAgICcP36dTx48AAAsH37dnzyyScaKdrBwQGdO3f+6Jp69+4NU1NT6X7fvn1hYmKC7du3v3e6t7dWvHr1CnFxcfjkk08AAKdPnwbw5viBTZs2oVWrVlrPonh3i1OvXr1gbGws3d+7dy9SU1MxePBgGBkZafSztrbGtm3bAED6D37Xrl1ITk7WWm/G/vvNmzdr3eWU3XVNSEhAXFwc/Pz8cOvWrQ/uSskOfY5vjx49NJ5nHx8fCCHQo0cPqc3Y2Bg1a9bErVu3Mk3fpk0buLi4SPdr164NHx8f6fVhbm4OpVKJAwcOZGuXaobo6GicPXsWISEhGltgmjRpAi8vL42+27dvh7GxMQYOHKjR/s0330AIgR07dgAANmzYALVajcDAQOn9FBcXBycnJ5QrVw779+8HkL3Xy/tYWlriq6++ku4rlUrUrl1b4/nbvn07nJyc0LFjR6nN1NQUAwcOxIsXL/D333/rvNwP1dShQwfpfoUKFWBrawtPT0+NLQ8Zf2sb63e3bg4YMEBal7f5+flpjFF6ejp2796NNm3aoEyZMlK7s7MzOnXqhEOHDiExMREAEBQUhNevX2PDhg1Sv927dyM+Ph5BQUEA3mwBXL9+PVq1agUhhMZYBgQEICEhQfqMeR9fX194e3tL90uVKoXPP/8cu3btQnp6eo6WExISkq0ttbo8Jxne/fzL8DFj+3atr1+/xpMnT1C2bFnY2tpqfQ579+6t8XlRv359pKen4+7duwDe7Al4/vw5hg8fnum4sIzpzp49i+vXr6NTp0548uSJ9JwmJSWhcePGOHjwoM6fwXmJQUjPXrx4IW2yvHHjBoQQGD16NBwcHDRu4eHhAN4cYwC82c9brly5TPOrUKHCR9f07nwtLS3h7Oz8weM7nj59ikGDBsHR0RHm5uZwcHBA6dKlAUAKB48fP0ZiYqK0GfpDMqbPkPHme3c9lUolypQpIz1eunRphIWF4ZdffoG9vT0CAgIwd+5cjZASFBSEunXromfPnnB0dESHDh2wdu3abL0hDx8+DH9/fxQpUgS2trZwcHCQji3KjSCkz/EtVaqUxv2MEODq6pqpXVuQ0VZX+fLlpdeHSqXCjz/+iB07dsDR0RGffvoppkyZgpiYmPfWlTF22Vnvu3fvokSJEpl25Xh6emrM6/r16xBCoFy5cpneU5cvX5beT9l5vbxPyZIlM4V4Ozs7jecvY0zfDvDaas4t2mqysbHROs4AsjXWHh4eMDIyyvRZ8O779PHjx0hOTtb6evX09IRarZaO0apWrRoqVqyosatlzZo1sLe3R6NGjaT5xcfHY9GiRZnGsVu3bgD+99n4Plm9dpOTk/H48eMcLefddc+KLs/Jh+b9MWP78uVLjBkzRjq2zt7eHg4ODoiPj9f6en/388LOzk5jnjdv3gSA936mX79+HcCb0Pju8/rLL78gJSUlVz439YXHCOnRf//9h4SEBJQtWxYApC/goUOHSvvF35XRNzekp6fn2ryAN8cTHDlyBN9++y2qV68OS0tLqNVqNGvWLMdp/2OOiZk+fTq6du2KzZs3Y/fu3Rg4cKB0rEPJkiVhbm6OgwcPYv/+/di2bRt27tyJNWvWoFGjRti9e7fW/8SAN2/8xo0bo2LFipgxYwZcXV2hVCqxfft2zJw5M9/8Z5PV+Ga1XtrahZYDj7Nj8ODBaNWqFTZt2oRdu3Zh9OjRiIiIwL59+1CjRo0czTMn1Go1FAoFduzYkeV/1hk+9Hp5n6ye05w+f7lBl3EGsldrVscIfuyxa0FBQZg4cSLi4uJgZWWFLVu2oGPHjjAxefMVlPGe+uqrrzIdS5QhNy5DkpPl6POM36zm/TFjO2DAACxduhSDBw+Gr6+vdMHHDh06aP3syo3XdsZ8p06diurVq2vt8/Z7Mb9hENKjlStXAoAUejI2l5qamsLf3/+907q5uUkp+21Xr17N1GZnZ5fpjKrU1FRER0drnff169fRsGFD6f6LFy8QHR2NFi1aZFnPs2fPEBkZiXHjxmHMmDEa83qbg4MDrK2tM51llF0ZZ2NcvXpVY/Nyamoqbt++nel5q1KlCqpUqYJRo0bhyJEjqFu3LhYsWIAJEyYAeHPqcePGjdG4cWPMmDEDkyZNwsiRI7F///4sx2Dr1q1ISUnBli1bNP5bytjNkhv0Ob4fS1td165dy3QQtIeHB7755ht88803uH79OqpXr47p06dnOksyQ8bYZme93dzcsHfvXjx//lxjq9CVK1c05uXh4QEhBEqXLo3y5ct/cN0+9Hr5GG5ubjh//jzUarXGVqF3a37fCQl57fr16xpbJW7cuAG1Wv3Bq3w7ODjAwsJC6+v1ypUrMDIy0th6ERQUhHHjxmH9+vVwdHREYmKixq4fBwcHWFlZIT09/YOfjR9an3ddu3YNFhYW0tl6ubEcbXR9TvRl3bp1CAkJwfTp06W2V69eZfoMyS4PDw8AwL///pvlP+oZfaytrXP9ec0L3DWmJ/v27cP48eNRunRp6biP4sWLo0GDBli4cKHWL7G3T1ls0aIFjh49iuPHj2s8vmrVqkzTeXh4ZDpFdNGiRVluMVi0aBFev34t3Z8/fz7S0tKks1+0yfiv4d3/EmbNmqVx38jICG3atMHWrVtx8uTJTPP50H8Z/v7+UCqV+PnnnzX6LlmyBAkJCWjZsiUAIDExEWlpaRrTVqlSBUZGRkhJSQHwZlfeuzL+W8noo422dU1ISMDSpUvfW7su9Dm+H2vTpk3SsWoAcPz4cRw7dkx6fSQnJ2c6xdrDwwNWVlbvfV6dnZ1RvXp1LF++PNMlDi5duqTRt0WLFkhPT8ecOXM02mfOnAmFQiHV8sUXX8DY2Bjjxo3L9NoSQkiXdsjO6+VjtWjRAjExMRq7gNLS0jB79mxYWlrCz88PAKSzdnL6xZSb5s6dq3F/9uzZAPDezwLgzXukadOm2Lx5s8ZutNjYWPz++++oV68erK2tpXZPT09UqVIFa9aswZo1a+Ds7IxPP/1UY37t2rXD+vXrtf4Tpe10bm2ioqI0joO5f/8+Nm/ejKZNm8LY2DjXlqONrs+JvhgbG2d6L8yePTvHnxdNmzaFlZUVIiIiMr3vM5bj7e0NDw8PTJs2DS9evMg0j495XvMCtwjlgh07duDKlStIS0tDbGws9u3bhz179sDNzQ1btmzROMBs7ty5qFevHqpUqYJevXqhTJkyiI2NRVRUFP777z+cO3cOAPDdd99h5cqVaNasGQYNGiSdXp3xX+fbevbsiT59+qBdu3Zo0qQJzp07h127dsHe3l5rvampqWjcuDECAwNx9epVzJs3D/Xq1UPr1q2zXEdra2vpWJDXr1/DxcUFu3fvxu3btzP1nTRpEnbv3g0/Pz/07t0bnp6eiI6Oxp9//olDhw699yJkDg4OGDFiBMaNG4dmzZqhdevWUo21atWSDljdt28f+vfvj/bt26N8+fJIS0vDypUrpQ864M0Vvw8ePIiWLVvCzc0Njx49wrx581CyZEnUq1cvyxqaNm0KpVKJVq1a4euvv8aLFy+wePFiFC9ePNe2wuhzfD9W2bJlUa9ePfTt2xcpKSmYNWsWihUrhu+++w7Am/+wM14/Xl5eMDExwcaNGxEbG6vxX742ERERaNmyJerVq4fu3bvj6dOnmD17NipVqqTxAdqqVSs0bNgQI0eOxJ07d1CtWjXs3r0bmzdvxuDBg6X/QD08PDBhwgSMGDECd+7cQZs2bWBlZYXbt29j48aN6N27N4YOHZqt18vH6t27NxYuXIiuXbvi1KlTcHd3x7p163D48GHMmjVL2rJlbm4OLy8vrFmzBuXLl0fRokVRuXLlbB9Xl5tu376N1q1bo1mzZoiKisJvv/2GTp06oVq1ah+cdsKECdJ1uvr16wcTExMsXLgQKSkpWq+xFBQUhDFjxsDMzAw9evTIdCzV5MmTsX//fvj4+KBXr17w8vLC06dPcfr0aezdu1frPzbvqly5MgICAjROnwcgXck7t5aTW8+JPnz22WdYuXIlbGxs4OXlhaioKOzduxfFihXL0fysra0xc+ZM9OzZE7Vq1UKnTp1gZ2eHc+fOITk5GcuXL4eRkRF++eUXNG/eHJUqVUK3bt3g4uKCBw8eYP/+/bC2tsbWrVtzeU1zUV6eolbYZJx+mHFTKpXCyclJNGnSRPz000/S6bLvunnzpggODhZOTk7C1NRUuLi4iM8++0ysW7dOo9/58+eFn5+fMDMzEy4uLmL8+PFiyZIlmU6vTk9PF8OGDRP29vbCwsJCBAQEiBs3bmR5evXff/8tevfuLezs7ISlpaXo3LmzePLkyQfX97///hNt27YVtra2wsbGRrRv3148fPhQ66nAd+/eFcHBwcLBwUGoVCpRpkwZERoaKlJSUjRqyerSA3PmzBEVK1YUpqamwtHRUfTt21c8e/ZMevzWrVuie/fuwsPDQ5iZmYmiRYuKhg0bir1790p9IiMjxeeffy5KlCghlEqlKFGihOjYsaO4du3aB9d1y5YtomrVqsLMzEy4u7uLH3/8Ufz666+Znvucnj4vhP7G993nNONU6XcvIxASEiKKFCmSqfapU6eK6dOnC1dXV6FSqUT9+vU1TqeOi4sToaGhomLFiqJIkSLCxsZG+Pj4iLVr135wnYUQYv369cLT01OoVCrh5eUlNmzYIEJCQjROnxdCiOfPn4shQ4aIEiVKCFNTU1GuXDkxdepUjdNz355nvXr1RJEiRUSRIkVExYoVRWhoqLh69aoQInuvl6z4+fmJSpUqZWrXVnNsbKzo1q2bsLe3F0qlUlSpUkXra+HIkSPC29tbKJXKD55Kn9Xp89pqcnNzEy1btszUDkCEhoZK9zNeE5cuXRJffvmlsLKyEnZ2dqJ///6ZTpF+d9q3nT59WgQEBAhLS0thYWEhGjZsKI4cOaK17/Xr16XPykOHDmntExsbK0JDQ4Wrq6swNTUVTk5OonHjxmLRokVa+2ur87fffhPlypUTKpVK1KhRQ+N502U5Gc+7tksuZHX6fHafk/d9/n3s2D579kx6DVpaWoqAgABx5cqVbH9eaHu9CfHmM7FOnTrC3NxcWFtbi9q1a4s//vhDo8+ZM2fEF198IYoVKyZUKpVwc3MTgYGBIjIyMlPd+YlCCAMe7Ud5atmyZejWrRtOnDjBHwikTO7cuYPSpUtj6tSpGDp0qKHLIT3KuGjp48eP9bZlMa8pFAqEhoZm2p1K9CE8RoiIiIhki0GIiIiIZItBiIiIiGTLoMcIHTx4EFOnTsWpU6cQHR2NjRs3ok2bNu+d5sCBAwgLC8PFixfh6uqKUaNGafwCNxEREVF2GXSLUFJSEqpVq5bpWhZZuX37Nlq2bImGDRvi7NmzGDx4MHr27Ildu3bpuVIiIiIqjPLNWWMKheKDW4SGDRuGbdu2aVwIq0OHDoiPj8fOnTvzoEoiIiIqTArUBRWjoqIyXb47ICAAgwcPznKalJQUjSvHqtVqPH36FMWKFctXl7onIiKirAkh8Pz5c5QoUSLTBTk/RoEKQjExMXB0dNRoy/jdmpcvX2r9AbuIiAiNq4oSERFRwXX//v0P/lCyLgpUEMqJESNGICwsTLqfkJCAUqVK4f79+3nyuy9ERET08RITE+Hq6qrxQ8y5oUAFIScnJ8TGxmq0xcbGwtraWuvWIABQqVRQqVSZ2q2trRmEiIiICpjcPqylQF1HyNfXF5GRkRpte/bsga+vr4EqIiIiooLMoEHoxYsXOHv2LM6ePQvgzenxZ8+exb179wC82a0VHBws9e/Tpw9u3bqF7777DleuXMG8efOwdu1aDBkyxBDlExERUQFn0CB08uRJ1KhRAzVq1AAAhIWFoUaNGhgzZgwAIDo6WgpFAFC6dGls27YNe/bsQbVq1TB9+nT88ssvCAgIMEj9REREVLDlm+sI5ZXExETY2NggISGBxwgRERHlofT0dLx+/TrLx5VKZZanxuvr+7tAHSxNREREBY8QAjExMYiPj39vPyMjI5QuXRpKpTJvCgODEBEREelZRggqXrw4LCwstJ75pVar8fDhQ0RHR6NUqVJ5dtFjBiEiIiLSm/T0dCkEFStW7L19HRwc8PDhQ6SlpcHU1DRP6itQp88TERFRwZJxTJCFhcUH+2bsEktPT9drTW9jECIiIiK9y86uLkP8BiiDEBEREckWgxARERHJFoMQERERyRaDEBEREelddq7fbIhrPDMIERERkd5knAafnJz8wb6pqakAAGNjY73W9DZeR4iIiIj0xtjYGLa2tnj06BEAvPeCio8fP4aFhQVMTPIunjAIERERkV45OTkBgBSGsmJkZJSnV5UGGISIiIhIzxQKBZydnVG8ePEc/+iqvjAIERERUZ4wNjbO0+N/soMHSxMREZFsMQgRERGRbDEIERERkWwxCBEREZFsMQgRERGRbDEIERERkWwxCBEREZFsMQgRERGRbDEIERERkWwxCBEREZFsMQgRERGRbDEIERERkWwxCBEREZFsMQgRERGRbDEIERERkWwxCBEREZFsMQgRERGRbDEIERERkWwxCBEREZFsMQgRERGRbDEIERERkWwxCBEREZFsMQgRERGRbDEIERERkWwxCBEREZFsMQgRERGRbDEIERERkWwxCBEREZFsMQgRERGRbDEIERERkWwxCBEREZFsMQgRERGRbDEIERERkWwxCBEREZFsMQgRERGRbDEIERERkWwxCBEREZFsMQgRERGRbDEIERERkWwxCBEREZFsMQgRERGRbDEIERERkWwxCBEREZFsMQgRERGRbDEIERERkWwxCBEREZFsMQgRERGRbDEIERERkWwxCBEREZFsGTwIzZ07F+7u7jAzM4OPjw+OHz/+3v6zZs1ChQoVYG5uDldXVwwZMgSvXr3Ko2qJiIioMDFoEFqzZg3CwsIQHh6O06dPo1q1aggICMCjR4+09v/9998xfPhwhIeH4/Lly1iyZAnWrFmD77//Po8rJyIiosLAoEFoxowZ6NWrF7p16wYvLy8sWLAAFhYW+PXXX7X2P3LkCOrWrYtOnTrB3d0dTZs2RceOHT+4FYmIiIhIG4MFodTUVJw6dQr+/v7/K8bICP7+/oiKitI6TZ06dXDq1Ckp+Ny6dQvbt29HixYtslxOSkoKEhMTNW5EREREAGBiqAXHxcUhPT0djo6OGu2Ojo64cuWK1mk6deqEuLg41KtXD0IIpKWloU+fPu/dNRYREYFx48blau1ERERUOBj8YGldHDhwAJMmTcK8efNw+vRpbNiwAdu2bcP48eOznGbEiBFISEiQbvfv38/DiomIiCg/M9gWIXt7exgbGyM2NlajPTY2Fk5OTlqnGT16NLp06YKePXsCAKpUqYKkpCT07t0bI0eOhJFR5lynUqmgUqlyfwWIiIiowDPYFiGlUglvb29ERkZKbWq1GpGRkfD19dU6TXJycqawY2xsDAAQQuivWCIiIiqUDLZFCADCwsIQEhKCmjVronbt2pg1axaSkpLQrVs3AEBwcDBcXFwQEREBAGjVqhVmzJiBGjVqwMfHBzdu3MDo0aPRqlUrKRARERERZZdBg1BQUBAeP36MMWPGICYmBtWrV8fOnTulA6jv3bunsQVo1KhRUCgUGDVqFB48eAAHBwe0atUKEydONNQqEBERUQGmEDLbp5SYmAgbGxskJCTA2tra0OUQERFRNujr+7tAnTVGRERElJsYhIiIiEi2GISIiIhIthiEiIiISLYYhIiIiEi2GISIiIhIthiEiIiISLYYhIiIiEi2GISIiIhIthiEiIiISLYYhIiIiEi2GISIiIhIthiEiIiISLYYhIiIiEi2GISIiIhIthiEiIiISLYYhIiIiEi2GISIiIhIthiEiIiISLYYhIiIiEi2GISIiIhIthiEiIiISLYYhIiIiEi2GISIiIhIthiEiIiISLYYhIiIiEi2GISIiIhIthiEiIiISLYYhIiIiEi2GISIiIhIthiEiIiISLYYhIiIiEi2GISIiIhIthiEiIiISLYYhIiIiEi2GISIiIhIthiEiIiISLYYhIiIiEi2GISIiIhIthiEiIiISLYYhIiIiEi2GISIiIhIthiEiIiISLYYhIiIiEi2GISIiIhIthiEiIiISLYYhIiIiEi2GISIiIhIthiEiIiISLYYhIiIiEi2GISIiIhIthiEiIiISLYYhIiIiEi2GISIiIhIthiEiIiISLYYhIiIiEi2GISIiIhIthiEiIiISLYYhIiIiEi2GISIiIhIthiEiIiISLYYhIiIiEi2GISIiIhItgwehObOnQt3d3eYmZnBx8cHx48ff2//+Ph4hIaGwtnZGSqVCuXLl8f27dvzqFoiIiIqTEwMufA1a9YgLCwMCxYsgI+PD2bNmoWAgABcvXoVxYsXz9Q/NTUVTZo0QfHixbFu3Tq4uLjg7t27sLW1zfviiYiIqMBTCCGEoRbu4+ODWrVqYc6cOQAAtVoNV1dXDBgwAMOHD8/Uf8GCBZg6dSquXLkCU1PTHC0zMTERNjY2SEhIgLW19UfVT0RERHlDX9/fBts1lpqailOnTsHf3/9/xRgZwd/fH1FRUVqn2bJlC3x9fREaGgpHR0dUrlwZkyZNQnp6epbLSUlJQWJiosaNiIiICDBgEIqLi0N6ejocHR012h0dHRETE6N1mlu3bmHdunVIT0/H9u3bMXr0aEyfPh0TJkzIcjkRERGwsbGRbq6urrm6HkRERFRwGfxgaV2o1WoUL14cixYtgre3N4KCgjBy5EgsWLAgy2lGjBiBhIQE6Xb//v08rJiIiIjyM4MdLG1vbw9jY2PExsZqtMfGxsLJyUnrNM7OzjA1NYWxsbHU5unpiZiYGKSmpkKpVGaaRqVSQaVS5W7xREREVCjkaIvQP//8g6+++gq+vr548OABAGDlypU4dOhQtuehVCrh7e2NyMhIqU2tViMyMhK+vr5ap6lbty5u3LgBtVottV27dg3Ozs5aQxARERHR++gchNavX4+AgACYm5vjzJkzSElJAQAkJCRg0qRJOs0rLCwMixcvxvLly3H58mX07dsXSUlJ6NatGwAgODgYI0aMkPr37dsXT58+xaBBg3Dt2jVs27YNkyZNQmhoqK6rQURERKT7rrEJEyZgwYIFCA4OxurVq6X2unXrvvegZW2CgoLw+PFjjBkzBjExMahevTp27twpHUB97949GBn9L6u5urpi165dGDJkCKpWrQoXFxcMGjQIw4YN03U1iIiIiHS/jpCFhQUuXboEd3d3WFlZ4dy5cyhTpgxu3boFLy8vvHr1Sl+15gpeR4iIiKjgyTfXEXJycsKNGzcytR86dAhlypTJlaKIiIiI8oLOQahXr14YNGgQjh07BoVCgYcPH2LVqlUYOnQo+vbtq48aiYiIiPRC52OEhg8fDrVajcaNGyM5ORmffvopVCoVhg4digEDBuijRiIiIiK90OkYofT0dBw+fBhVq1aFhYUFbty4gRcvXsDLywuWlpb6rDPX8BghIiKigkdf3986bREyNjZG06ZNcfnyZdja2sLLyyvXCiEiIiLKazofI1S5cmXcunVLH7UQERER5Smdg9CECRMwdOhQ/PXXX4iOjuYvuxMREVGBpfN1hN6+wKFCoZD+FkJAoVAgPT0996rTAx4jREREVPDki2OEAGD//v25tnAiIiIiQ9I5CPn5+emjDiIiIqI8p3MQAoD4+HgsWbIEly9fBgBUqlQJ3bt3h42NTa4WR0RERKRPOh8sffLkSXh4eGDmzJl4+vQpnj59ihkzZsDDwwOnT5/WR41EREREeqHzwdL169dH2bJlsXjxYpiYvNmglJaWhp49e+LWrVs4ePCgXgrNLTxYmoiIqODR1/e3zkHI3NwcZ86cQcWKFTXaL126hJo1ayI5OTnXitMHBiEiIqKCJ9/8+ry1tTXu3buXqf3+/fuwsrLKlaKIiIiI8oLOQSgoKAg9evTAmjVrcP/+fdy/fx+rV69Gz5490bFjR33USERERKQXOp81Nm3aNCgUCgQHByMtLQ0AYGpqir59+2Ly5Mm5XiARERGRvuh8jFCG5ORk3Lx5EwDg4eEBCwuLXC1MX3iMEBERUcGTb64snZCQgPT0dBQtWhRVqlSR2p8+fQoTExOGCyIiIiowdD5GqEOHDli9enWm9rVr16JDhw65UhQRERFRXtA5CB07dgwNGzbM1N6gQQMcO3YsV4oiIiIiygs6B6GUlBTpIOm3vX79Gi9fvsyVooiIiIjygs5BqHbt2li0aFGm9gULFsDb2ztXiiIiIiLKCzofLD1hwgT4+/vj3LlzaNy4MQAgMjISJ06cwO7du3O9QCIiIiJ90XmLUN26dREVFQVXV1esXbsWW7duRdmyZXH+/HnUr19fHzUSERER6UWOryNUUPE6QkRERAWPwa8jlJaWhvT0dKhUKqktNjYWCxYsQFJSElq3bo169erlWmFERERE+pbtINSrVy8olUosXLgQAPD8+XPUqlULr169grOzM2bOnInNmzejRYsWeiuWiIiIKDdl+xihw4cPo127dtL9FStWID09HdevX8e5c+cQFhaGqVOn6qVIIiIiIn3IdhB68OABypUrJ92PjIxEu3btYGNjAwAICQnBxYsXc79CIiIiIj3JdhAyMzPTuGDi0aNH4ePjo/H4ixcvcrc6IiIiIj3KdhCqXr06Vq5cCQD4559/EBsbi0aNGkmP37x5EyVKlMj9ComIiIj0JNsHS48ZMwbNmzfH2rVrER0dja5du8LZ2Vl6fOPGjahbt65eiiQiIiLSh2wHIT8/P5w6dQq7d++Gk5MT2rdvr/F49erVUbt27VwvkIiIiEhfeEFFIiIiyvf09f2t809sEBERERUWDEJEREQkWwxCREREJFsMQkRERCRbDEJEREQkW9k+fT6DnZ0dFApFpnaFQgEzMzOULVsWXbt2Rbdu3XKlQCIiIiJ90TkIjRkzBhMnTkTz5s2l6wYdP34cO3fuRGhoKG7fvo2+ffsiLS0NvXr1yvWCiYiIiHKLzkHo0KFDmDBhAvr06aPRvnDhQuzevRvr169H1apV8fPPPzMIERERUb6m8zFCu3btgr+/f6b2xo0bY9euXQCAFi1a4NatWx9fHREREZEe6RyEihYtiq1bt2Zq37p1K4oWLQoASEpKgpWV1cdXR0RERKRHOu8aGz16NPr27Yv9+/dLxwidOHEC27dvx4IFCwAAe/bsgZ+fX+5WSkRERJTLcvRbY4cPH8acOXNw9epVAECFChUwYMAA1KlTJ9cLzG38rTEiIqKCR1/f3/zRVSIiIsr39PX9rfOuscTERK3tCoUCKpUKSqXyo4siIiIiygs6ByFbW1utF1TMULJkSXTt2hXh4eEwMuKFq4mIiCj/0jkILVu2DCNHjkTXrl01Lqi4fPlyjBo1Co8fP8a0adOgUqnw/fff53rBRERERLlF5yC0fPlyTJ8+HYGBgVJbq1atUKVKFSxcuBCRkZEoVaoUJk6cyCBERERE+ZrO+66OHDmCGjVqZGqvUaMGoqKiAAD16tXDvXv3Pr46IiIiIj3SOQi5urpiyZIlmdqXLFkCV1dXAMCTJ09gZ2f38dURERER6ZHOu8amTZuG9u3bY8eOHahVqxYA4OTJk7hy5QrWrVsH4M0FFoOCgnK3UiIiIqJclqPrCN25cwcLFy7UuKDi119/DXd399yuL9fxOkJEREQFT4G4oOK///6LypUr59bs9IJBiIiIqODR1/f3R1/o5/nz51i0aBFq166NatWq5UZNRERERHkix0Ho4MGDCAkJgbOzM6ZNm4ZGjRrh6NGjuVkbERERkV7pdLB0TEwMli1bhiVLliAxMRGBgYFISUnBpk2b4OXlpa8aiYiIiPQi21uEWrVqhQoVKuD8+fOYNWsWHj58iNmzZ+uzNiIiIiK9yvYWoR07dmDgwIHo27cvypUrp8+aiIiIiPJEtrcIHTp0CM+fP4e3tzd8fHwwZ84cxMXF6bM2IiIiIr3KdhD65JNPsHjxYkRHR+Prr7/G6tWrUaJECajVauzZswfPnz/PcRFz586Fu7s7zMzM4OPjg+PHj2drutWrV0OhUKBNmzY5XjYRERHJl85njRUpUgTdu3fHoUOHcOHCBXzzzTeYPHkyihcvjtatW+tcwJo1axAWFobw8HCcPn0a1apVQ0BAAB49evTe6e7cuYOhQ4eifv36Oi+TiIiICPjI6whVqFABU6ZMwX///Yc//vgjR/OYMWMGevXqhW7dusHLywsLFiyAhYUFfv311yynSU9PR+fOnTFu3DiUKVMmp+UTERGRzH30BRUBwNjYGG3atMGWLVt0mi41NRWnTp2Cv7///woyMoK/v7/0S/ba/PDDDyhevDh69OjxwWWkpKQgMTFR40ZEREQE5FIQyqm4uDikp6fD0dFRo93R0RExMTFapzl06BCWLFmCxYsXZ2sZERERsLGxkW6urq4fXTcREREVDgYNQrp6/vw5unTpgsWLF8Pe3j5b04wYMQIJCQnS7f79+3qukoiIiAoKna4sndvs7e1hbGyM2NhYjfbY2Fg4OTll6n/z5k3cuXMHrVq1ktrUajUAwMTEBFevXoWHh4fGNCqVCiqVSg/VExERUUFn0C1CSqUS3t7eiIyMlNrUajUiIyPh6+ubqX/FihVx4cIFnD17Vrq1bt0aDRs2xNmzZ7nbi4iIiHRi0C1CABAWFoaQkBDUrFkTtWvXxqxZs5CUlIRu3boBAIKDg+Hi4oKIiAiYmZmhcuXKGtPb2toCQKZ2IiIiog8xeBAKCgrC48ePMWbMGMTExKB69erYuXOndAD1vXv3YGRUoA5lIiIiogJCIYQQhi4iLyUmJsLGxgYJCQmwtrY2dDlERESUDfr6/uamFiIiIpItBiEiIiKSLQYhIiIiki0GISIiIpItBiEiIiKSLQYhIiIiki0GISIiIpItBiEiIiKSLQYhIiIiki0GISIiIpItBiEiIiKSLQYhIiIiki0GISIiIpItBiEiIiKSLQYhIiIiki0GISIiIpItBiEiIiKSLQYhIiIiki0GISIiIpItBiEiIiKSLQYhIiIiki0GISIiIpItBiEiIiKSLQYhIiIiki0GISIiIpItBiEiIiKSLQYhIiIiki0GISIiIpItBiEiIiKSLQYhIiIiki0GISIiIpItBiEiIiKSLQYhIiIiki0GISIiIpItBiEiIiKSLQYhIiIiki0GISIiIpItBiEiIiKSLQYhIiIiki0GISIiIpItBiEiIiKSLQYhIiIiki0GISIiIpItBiEiIiKSLQYhIiIiki0GISIiIpItBiEiIiKSLQYhIiIiki0GISIiIpItBiEiIiKSLQYhIiIiki0GISIiIpItBiEiIiKSLQYhIiIiki0GISIiIpItBiEiIiKSLQYhIiIiki0GISIiIpItBiEiIiKSLQYhIiIiki0GISIiIpItBiEiIiKSLQYhIiIikq18EYTmzp0Ld3d3mJmZwcfHB8ePH8+y7+LFi1G/fn3Y2dnBzs4O/v7+7+1PRERElBWDB6E1a9YgLCwM4eHhOH36NKpVq4aAgAA8evRIa/8DBw6gY8eO2L9/P6KiouDq6oqmTZviwYMHeVw5ERERFXQKIYQwZAE+Pj6oVasW5syZAwBQq9VwdXXFgAEDMHz48A9On56eDjs7O8yZMwfBwcEf7J+YmAgbGxskJCTA2tr6o+snIiIi/dPX97dBtwilpqbi1KlT8Pf3l9qMjIzg7++PqKiobM0jOTkZr1+/RtGiRbU+npKSgsTERI0bEREREWDgIBQXF4f09HQ4OjpqtDs6OiImJiZb8xg2bBhKlCihEabeFhERARsbG+nm6ur60XUTERFR4WDwY4Q+xuTJk7F69Wps3LgRZmZmWvuMGDECCQkJ0u3+/ft5XCURERHlVyaGXLi9vT2MjY0RGxur0R4bGwsnJ6f3Tjtt2jRMnjwZe/fuRdWqVbPsp1KpoFKpcqVeIiIiKlwMukVIqVTC29sbkZGRUptarUZkZCR8fX2znG7KlCkYP348du7ciZo1a+ZFqURERFQIGXSLEACEhYUhJCQENWvWRO3atTFr1iwkJSWhW7duAIDg4GC4uLggIiICAPDjjz9izJgx+P333+Hu7i4dS2RpaQlLS0uDrQcREREVPAYPQkFBQXj8+DHGjBmDmJgYVK9eHTt37pQOoL537x6MjP634Wr+/PlITU3Fl19+qTGf8PBwjB07Ni9LJyIiogLO4NcRymu8jhAREVHBUyivI0RERERkSAxCREREJFsMQkRERCRbDEJEREQkWwxCREREJFsMQkRERCRbDEJEREQkWwxCREREJFsMQkRERCRbDEJEREQkWwxCREREJFsMQkRERCRbDEJEREQkWwxCREREJFsMQkRERCRbDEJEREQkWwxCREREJFsMQkRERCRbDEJEREQkWwxCREREJFsMQkRERCRbDEJEREQkWwxCREREJFsMQkRERCRbDEJEREQkWwxCREREJFsMQkRERCRbDEJEREQkWwxCREREJFsMQkRERCRbDEJEREQkWwxCREREJFsMQkRERCRbDEJEREQkWwxCREREJFsMQkRERCRbDEJEREQkWwxCREREJFsMQkRERCRbDEJEREQkWwxCREREJFsMQkRERCRbDEJEREQkWwxCREREJFsMQkRERCRbDEJEREQkWwxCREREJFsMQkRERCRbDEJEREQkWwxCREREJFsMQkRERCRbDEJEREQkWwxCREREJFsMQkRERCRbDEJEREQkWwxCREREJFsMQkRERCRbDEJEREQkWwxCREREJFsMQkRERCRbDEJEREQkWwxCREREJFv5IgjNnTsX7u7uMDMzg4+PD44fP/7e/n/++ScqVqwIMzMzVKlSBdu3b8+jSomIiKgwMXgQWrNmDcLCwhAeHo7Tp0+jWrVqCAgIwKNHj7T2P3LkCDp27IgePXrgzJkzaNOmDdq0aYN///03jysnIiKigk4hhBCGLMDHxwe1atXCnDlzAABqtRqurq4YMGAAhg8fnql/UFAQkpKS8Ndff0ltn3zyCapXr44FCxZ8cHmJiYmwsbFBQkICrK2tc29FiIiISG/09f1t0C1CqampOHXqFPz9/aU2IyMj+Pv7IyoqSus0UVFRGv0BICAgIMv+RERERFkxMeTC4+LikJ6eDkdHR412R0dHXLlyRes0MTExWvvHxMRo7Z+SkoKUlBTpfkJCAoA3yZKIiIgKhozv7dzekWXQIJQXIiIiMG7cuEztrq6uBqiGiIiIPsaTJ09gY2OTa/MzaBCyt7eHsbExYmNjNdpjY2Ph5OSkdRonJyed+o8YMQJhYWHS/fj4eLi5ueHevXu5+kSS7hITE+Hq6or79+/zeK18gOORf3As8g+ORf6RkJCAUqVKoWjRork6X4MGIaVSCW9vb0RGRqJNmzYA3hwsHRkZif79+2udxtfXF5GRkRg8eLDUtmfPHvj6+mrtr1KpoFKpMrXb2NjwRZ1PWFtbcyzyEY5H/sGxyD84FvmHkVHuHt5s8F1jYWFhCAkJQc2aNVG7dm3MmjULSUlJ6NatGwAgODgYLi4uiIiIAAAMGjQIfn5+mD59Olq2bInVq1fj5MmTWLRokSFXg4iIiAoggwehoKAgPH78GGPGjEFMTAyqV6+OnTt3SgdE37t3TyP91alTB7///jtGjRqF77//HuXKlcOmTZtQuXJlQ60CERERFVAGD0IA0L9//yx3hR04cCBTW/v27dG+ffscLUulUiE8PFzr7jLKWxyL/IXjkX9wLPIPjkX+oa+xMPgFFYmIiIgMxeA/sUFERERkKAxCREREJFsMQkRERCRbDEJEREQkW4UyCM2dOxfu7u4wMzODj48Pjh8//t7+f/75JypWrAgzMzNUqVIF27dvz6NKCz9dxmLx4sWoX78+7OzsYGdnB39//w+OHelG1/dGhtWrV0OhUEgXPqWPp+tYxMfHIzQ0FM7OzlCpVChfvjw/q3KJrmMxa9YsVKhQAebm5nB1dcWQIUPw6tWrPKq28Dp48CBatWqFEiVKQKFQYNOmTR+c5sCBA/i///s/qFQqlC1bFsuWLdN9waKQWb16tVAqleLXX38VFy9eFL169RK2trYiNjZWa//Dhw8LY2NjMWXKFHHp0iUxatQoYWpqKi5cuJDHlRc+uo5Fp06dxNy5c8WZM2fE5cuXRdeuXYWNjY3477//8rjywknX8chw+/Zt4eLiIurXry8+//zzvCm2kNN1LFJSUkTNmjVFixYtxKFDh8Tt27fFgQMHxNmzZ/O48sJH17FYtWqVUKlUYtWqVeL27dti165dwtnZWQwZMiSPKy98tm/fLkaOHCk2bNggAIiNGze+t/+tW7eEhYWFCAsLE5cuXRKzZ88WxsbGYufOnTott9AFodq1a4vQ0FDpfnp6uihRooSIiIjQ2j8wMFC0bNlSo83Hx0d8/fXXeq1TDnQdi3elpaUJKysrsXz5cn2VKCs5GY+0tDRRp04d8csvv4iQkBAGoVyi61jMnz9flClTRqSmpuZVibKh61iEhoaKRo0aabSFhYWJunXr6rVOuclOEPruu+9EpUqVNNqCgoJEQECATssqVLvGUlNTcerUKfj7+0ttRkZG8Pf3R1RUlNZpoqKiNPoDQEBAQJb9KXtyMhbvSk5OxuvXr3P9B/bkKKfj8cMPP6B48eLo0aNHXpQpCzkZiy1btsDX1xehoaFwdHRE5cqVMWnSJKSnp+dV2YVSTsaiTp06OHXqlLT77NatW9i+fTtatGiRJzXT/+TW93e+uLJ0bomLi0N6err08xwZHB0dceXKFa3TxMTEaO0fExOjtzrlICdj8a5hw4ahRIkSmV7opLucjMehQ4ewZMkSnD17Ng8qlI+cjMWtW7ewb98+dO7cGdu3b8eNGzfQr18/vH79GuHh4XlRdqGUk7Ho1KkT4uLiUK9ePQghkJaWhj59+uD777/Pi5LpLVl9fycmJuLly5cwNzfP1nwK1RYhKjwmT56M1atXY+PGjTAzMzN0ObLz/PlzdOnSBYsXL4a9vb2hy5E9tVqN4sWLY9GiRfD29kZQUBBGjhyJBQsWGLo02Tlw4AAmTZqEefPm4fTp09iwYQO2bduG8ePHG7o0yqFCtUXI3t4exsbGiI2N1WiPjY2Fk5OT1mmcnJx06k/Zk5OxyDBt2jRMnjwZe/fuRdWqVfVZpmzoOh43b97EnTt30KpVK6lNrVYDAExMTHD16lV4eHjot+hCKifvDWdnZ5iamsLY2Fhq8/T0RExMDFJTU6FUKvVac2GVk7EYPXo0unTpgp49ewIAqlSpgqSkJPTu3RsjR47U+JFw0q+svr+tra2zvTUIKGRbhJRKJby9vREZGSm1qdVqREZGwtfXV+s0vr6+Gv0BYM+ePVn2p+zJyVgAwJQpUzB+/Hjs3LkTNWvWzItSZUHX8ahYsSIuXLiAs2fPSrfWrVujYcOGOHv2LFxdXfOy/EIlJ++NunXr4saNG1IYBYBr167B2dmZIegj5GQskpOTM4WdjIAq+NOdeSrXvr91O447/1u9erVQqVRi2bJl4tKlS6J3797C1tZWxMTECCGE6NKlixg+fLjU//Dhw8LExERMmzZNXL58WYSHh/P0+Vyi61hMnjxZKJVKsW7dOhEdHS3dnj9/bqhVKFR0HY938ayx3KPrWNy7d09YWVmJ/v37i6tXr4q//vpLFC9eXEyYMMFQq1Bo6DoW4eHhwsrKSvzxxx/i1q1bYvfu3cLDw0MEBgYaahUKjefPn4szZ86IM2fOCABixowZ4syZM+Lu3btCCCGGDx8uunTpIvXPOH3+22+/FZcvXxZz587l6fMZZs+eLUqVKiWUSqWoXbu2OHr0qPSYn5+fCAkJ0ei/du1aUb58eaFUKkWlSpXEtm3b8rjiwkuXsXBzcxMAMt3Cw8PzvvBCStf3xtsYhHKXrmNx5MgR4ePjI1QqlShTpoyYOHGiSEtLy+OqCyddxuL169di7NixwsPDQ5iZmQlXV1fRr18/8ezZs7wvvJDZv3+/1u+AjOc/JCRE+Pn5ZZqmevXqQqlUijJlyoilS5fqvFyFENyWR0RERPJUqI4RIiIiItIFgxARERHJFoMQERERyRaDEBEREckWgxARERHJFoMQERERyRaDEBEREckWgxBRIeXu7o5Zs2Zlu/+BAwegUCgQHx+vt5o+xrJly2Bra2voMjKJjIyEp6cn0tPTAQBjx45F9erV3ztN165d0aZNG+l+gwYNMHjwYOn+u2OnUCiwadOm3Cv6Azp06IDp06fn2fKIDIlBiMjAFArFe29jx47N0XxPnDiB3r17Z7t/nTp1EB0dDRsbmxwtL7vya6DJqe+++w6jRo3S+EHUD/npp5+wbNmybPePjo5G8+bNc1BdzowaNQoTJ05EQkJCni2TyFAK1a/PExVE0dHR0t9r1qzBmDFjcPXqVanN0tJS+lsIgfT0dJiYfPit6+DgoFMdSqUyy1/cJu0OHTqEmzdvol27djpNp2vYzOtxqVy5Mjw8PPDbb78hNDQ0T5dNlNe4RYjIwJycnKSbjY0NFAqFdP/KlSuwsrLCjh074O3tDZVKJX35fv7553B0dISlpSVq1aqFvXv3asxX2+6VX375BW3btoWFhQXKlSuHLVu2SI+/u2ssY8vNrl274OnpCUtLSzRr1kwjuKWlpWHgwIGwtbVFsWLFMGzYMISEhGjs9nnbgQMH0K1bNyQkJGTa4vXs2TMEBwfDzs4OFhYWaN68Oa5fv57l8/b48WPUrFkTbdu2RUpKCtRqNSIiIlC6dGmYm5ujWrVqWLduXab1i4yMRM2aNWFhYYE6depohM5z586hYcOGsLKygrW1Nby9vXHy5Mksa1i9ejWaNGkCMzOzTI8tXLgQrq6usLCwQGBgoMbWlXd3jX3Iu7vGLly4gEaNGsHc3BzFihVD79698eLFi0zznzZtGpydnVGsWDGEhobi9evXUp958+ahXLlyMDMzg6OjI7788kuNZbZq1QqrV6/Odo1EBRWDEFEBMHz4cEyePBmXL19G1apV8eLFC7Ro0QKRkZE4c+YMmjVrhlatWuHevXvvnc+4ceMQGBiI8+fPo0WLFujcuTOePn2aZf/k5GRMmzYNK1euxMGDB3Hv3j0MHTpUevzHH3/EqlWrsHTpUhw+fBiJiYnvPZalTp06mDVrFqytrREdHY3o6Ghpfl27dsXJkyexZcsWREVFQQiBFi1aaHx5Z7h//z7q16+PypUrY926dVCpVIiIiMCKFSuwYMECXLx4EUOGDMFXX32Fv//+W2PakSNHYvr06Th58iRMTEzQvXt36bHOnTujZMmSOHHiBE6dOoXhw4fD1NQ0y/X5559/ULNmzUztN27cwNq1a7F161bs3LkTZ86cQb9+/bKcjy6SkpIQEBAAOzs7nDhxAn/++Sf27t2L/v37a/Tbv38/bt68if3792P58uVYtmyZtDvu5MmTGDhwIH744QdcvXoVO3fuxKeffqoxfe3atXH8+HGkpKTkSt1E+dbH/VYsEeWmpUuXChsbG+l+xq8xb9q06YPTVqpUScyePVu67+bmJmbOnCndByBGjRol3X/x4oUAIHbs2KGxrIxf0V66dKkAIG7cuCFNM3fuXOHo6Cjdd3R0FFOnTpXup6WliVKlSr33V+rfXUchhLh27ZoAIA4fPiy1xcXFCXNzc7F27VqN6a5cuSJcXV3FwIEDhVqtFkII8erVK2FhYSGOHDmiMd8ePXqIjh07aqzf3r17pce3bdsmAIiXL18KIYSwsrISy5Yty7L2d9nY2IgVK1ZotIWHhwtjY2Px33//SW07duwQRkZGIjo6Wgjx5le0336O/Pz8xKBBg6T72sZu48aNQgghFi1aJOzs7MSLFy801sPIyEjExMRI83dzc9P4dfr27duLoKAgIYQQ69evF9bW1iIxMTHLdTt37pwAIO7cuZO9J4OogOIWIaIC4N2tDi9evMDQoUPh6ekJW1tbWFpa4vLlyx/cIlS1alXp7yJFisDa2hqPHj3Ksr+FhQU8PDyk+87OzlL/hIQExMbGonbt2tLjxsbG8Pb21mndAODy5cswMTGBj4+P1FasWDFUqFABly9fltpevnyJ+vXr44svvsBPP/0EhUIB4M0WmOTkZDRp0gSWlpbSbcWKFbh582aWz4GzszMASOsUFhaGnj17wt/fH5MnT8407btevnypdbdYqVKl4OLiIt339fWFWq3W2A2XU5cvX0a1atVQpEgRqa1u3bqZ5l+pUiWNA7jfHrsmTZrAzc0NZcqUQZcuXbBq1SokJydrLMfc3BwAMrUTFTYMQkQFwNtfegAwdOhQbNy4EZMmTcI///yDs2fPokqVKkhNTX3vfN7dzaNQKKBWq3XqL4TQsfrco1Kp4O/vj7/++gsPHjyQ2jOOj9m2bRvOnj0r3S5duqRxnBCguU4ZQSrjORg7diwuXryIli1bYt++ffDy8sLGjRuzrMfe3h7Pnj3LtfXLTe8baysrK5w+fRp//PEHnJ2dMWbMGFSrVk3j0gkZu0x1PeieqKBhECIqgA4fPoyuXbuibdu2qFKlCpycnHDnzp08rcHGxgaOjo44ceKE1Jaeno7Tp0+/dzqlUildcyeDp6cn0tLScOzYMantyZMnuHr1Kry8vKQ2IyMjrFy5Et7e3mjYsCEePnwIAPDy8oJKpcK9e/dQtmxZjZurq6tO61W+fHkMGTIEu3fvxhdffIGlS5dm2bdGjRq4dOlSpvZ79+5JtQHA0aNHYWRkhAoVKuhUizaenp44d+4ckpKSpLbDhw/rPH8TExP4+/tjypQpOH/+PO7cuYN9+/ZJj//7778oWbIk7O3tP7pmovyMQYioACpXrhw2bNiAs2fP4ty5c+jUqdN7t+zoy4ABAxAREYHNmzfj6tWrGDRoEJ49eyZtadHG3d0dL168QGRkJOLi4pCcnIxy5crh888/R69evXDo0CGcO3cOX331FVxcXPD5559rTG9sbIxVq1ahWrVqaNSoEWJiYmBlZYWhQ4diyJAhWL58OW7evInTp09j9uzZWL58ebbW5eXLl+jfvz8OHDiAu3fv4vDhwzhx4gQ8PT2znCYgIACHDh3K1G5mZoaQkBCcO3cO//zzDwYOHIjAwMBcOQ2+c+fO0vz//fdf7N+/HwMGDECXLl3g6OiYrXn89ddf+Pnnn3H27FncvXsXK1asgFqt1ghS//zzD5o2bfrR9RLldwxCRAXQjBkzYGdnhzp16qBVq1YICAjA//3f/+V5HcOGDUPHjh0RHBwMX19fWFpaIiAgQOtxMxnq1KmDPn36ICgoCA4ODpgyZQoAYOnSpfD29sZnn30GX19fCCGwfft2rWdtmZiY4I8//kClSpXQqFEjPHr0COPHj8fo0aMREREBT09PNGvWDNu2bUPp0qWztS7GxsZ48uQJgoODUb58eQQGBqJ58+YYN25cltN07twZFy9ezHTsT9myZfHFF1+gRYsWaNq0KapWrYp58+Zlq44PsbCwwK5du/D06VPUqlULX375JRo3bow5c+Zkex62trbYsGEDGjVqBE9PTyxYsEB6PgHg1atX2LRpE3r16pUrNRPlZwphyB3+RFSoqNVqeHp6IjAwEOPHjzd0OXni22+/RWJiIhYuXGjoUnLN/PnzsXHjRuzevdvQpRDpHbcIEVGO3b17F4sXL8a1a9dw4cIF9O3bF7dv30anTp0MXVqeGTlyJNzc3Ayya1JfTE1NMXv2bEOXQZQnuEWIiHLs/v376NChA/79918IIVC5cmVMnjw508X5iIjyKwYhIiIiki3uGiMiIiLZYhAiIiIi2WIQIiIiItliECIiIiLZYhAiIiIi2WIQIiIiItliECIiIiLZYhAiIiIi2WIQIiIiItn6f+4gJ8BjIYIBAAAAAElFTkSuQmCC",
+      "text/plain": [
+       "<Figure size 640x480 with 1 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from matplotlib import pyplot as plt\n",
+    "from matplotlib import pyplot as plt\n",
+    "\n",
+    "import json\n",
+    "import os\n",
+    "from matplotlib import pyplot as plt\n",
+    "metrics = ['agg_score', 'commonsense_qa/acc_norm', 'hellaswag/acc_norm', 'openbookqa/acc_norm', 'piqa/acc_norm',\n",
+    "                   'siqa/acc_norm', 'winogrande/acc_norm', 'arc/acc_norm', 'mmlu/acc_norm']\n",
+    "\n",
+    "def normalize_runname(runname):\n",
+    "    return runname.replace(\"/\", \"_\")\n",
+    "\n",
+    "grouped = (\n",
+    "    df.groupby([\"runname\", \"steps\"])\n",
+    "    .agg(\n",
+    "        {\n",
+    "            key: \"mean\" for key in metrics\n",
+    "        }\n",
+    "    )\n",
+    "    .reset_index()\n",
+    ")\n",
+    "\n",
+    "file_id=\"../assets/data/plots/all_dumps_bad\"\n",
+    "files = {}\n",
+    "for metric in metrics:\n",
+    "    datas = {}\n",
+    "    for name, group in grouped.groupby(\"runname\"):\n",
+    "        # if name not in runs_mapping:\n",
+    "        #     continue\n",
+    "        group = group[[\"steps\", metric]].sort_values(by=\"steps\")\n",
+    "        group = group.set_index(\"steps\")\n",
+    "        rolling_avg = group\n",
+    "        # rolling_avg = group.rolling(window=5).mean()\n",
+    "        datas[name] = {\n",
+    "            \"x\": (rolling_avg.index * 2048 * 1024 * 1e-9).tolist(),\n",
+    "            \"y\": rolling_avg[metric].tolist(),\n",
+    "            \"label\": runs_mapping[name],\n",
+    "        }\n",
+    "    # Sort the datata based on the steps\n",
+    "    datas = {k: v for k, v in sorted(datas.items(), key=lambda x: -x[1][\"y\"][-1])}\n",
+    "    # Create a folder\n",
+    "    os.makedirs(f\"{file_id}\", exist_ok=True)\n",
+    "    with open(f\"{file_id}/{normalize_runname(metric)}.json\", \"w\") as f:\n",
+    "        json.dump({\n",
+    "            \"data\": datas,\n",
+    "            \"layout\": {\n",
+    "                \"title\": {\n",
+    "                    \"text\": \"Dedup across all dumps does not improve performance\"\n",
+    "                },\n",
+    "            }\n",
+    "        }, f)\n",
+    "    files[metric] = {\"file\": f\"{normalize_runname(metric)}.json\"}\n",
+    "# Create index\n",
+    "with open(f\"{file_id}/index.json\", \"w\") as f:\n",
+    "    json.dump({\n",
+    "        \"files\": files,\n",
+    "        \"settings\": {\n",
+    "            \"defaultMetric\": \"agg_score\",\n",
+    "            \"slider\":{\"min\":0,\"max\":30,\"default\":5}\n",
+    "        }\n",
+    "    }, f)\n",
+    "# Add labels and legend\n",
+    "plt.xlabel('Training tokens (billions)')\n",
+    "plt.ylabel('Agg Score')\n",
+    "plt.title('Dedup across all dumps does not improve performance')\n",
+    "plt.legend()\n",
+    "\n",
+    "# Show the plot\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "af28ebbd054cdc33",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-04-30T15:07:36.363849Z",
+     "start_time": "2024-04-30T15:07:36.362222Z"
+    },
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/plot_dedup_attempts.ipynb

@@ -0,0 +1,578 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "138889b92720ce2e",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-04-30T15:08:02.398435Z",
+     "start_time": "2024-04-30T15:08:02.194901Z"
+    },
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>runname</th>\n",
+       "      <th>seed</th>\n",
+       "      <th>steps</th>\n",
+       "      <th>agg_score</th>\n",
+       "      <th>commonsense_qa/acc</th>\n",
+       "      <th>commonsense_qa/acc_norm</th>\n",
+       "      <th>hellaswag/acc</th>\n",
+       "      <th>hellaswag/acc_norm</th>\n",
+       "      <th>openbookqa/acc</th>\n",
+       "      <th>openbookqa/acc_norm</th>\n",
+       "      <th>...</th>\n",
+       "      <th>siqa/acc</th>\n",
+       "      <th>siqa/acc_norm</th>\n",
+       "      <th>winogrande/acc</th>\n",
+       "      <th>winogrande/acc_norm</th>\n",
+       "      <th>sciq/acc</th>\n",
+       "      <th>sciq/acc_norm</th>\n",
+       "      <th>arc/acc</th>\n",
+       "      <th>arc/acc_norm</th>\n",
+       "      <th>mmlu/acc</th>\n",
+       "      <th>mmlu/acc_norm</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>big-run-refinedweb</td>\n",
+       "      <td>6</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.330893</td>\n",
+       "      <td>0.186</td>\n",
+       "      <td>0.233</td>\n",
+       "      <td>0.272</td>\n",
+       "      <td>0.258</td>\n",
+       "      <td>0.166</td>\n",
+       "      <td>0.286</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.367</td>\n",
+       "      <td>0.362</td>\n",
+       "      <td>0.516</td>\n",
+       "      <td>0.497</td>\n",
+       "      <td>0.208</td>\n",
+       "      <td>0.202</td>\n",
+       "      <td>0.2195</td>\n",
+       "      <td>0.2510</td>\n",
+       "      <td>0.230294</td>\n",
+       "      <td>0.250147</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>big-run-refinedweb</td>\n",
+       "      <td>6</td>\n",
+       "      <td>1000</td>\n",
+       "      <td>0.353481</td>\n",
+       "      <td>0.233</td>\n",
+       "      <td>0.253</td>\n",
+       "      <td>0.288</td>\n",
+       "      <td>0.276</td>\n",
+       "      <td>0.120</td>\n",
+       "      <td>0.256</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.365</td>\n",
+       "      <td>0.398</td>\n",
+       "      <td>0.502</td>\n",
+       "      <td>0.500</td>\n",
+       "      <td>0.582</td>\n",
+       "      <td>0.528</td>\n",
+       "      <td>0.2650</td>\n",
+       "      <td>0.2900</td>\n",
+       "      <td>0.240583</td>\n",
+       "      <td>0.252852</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>big-run-refinedweb</td>\n",
+       "      <td>6</td>\n",
+       "      <td>2000</td>\n",
+       "      <td>0.376461</td>\n",
+       "      <td>0.282</td>\n",
+       "      <td>0.280</td>\n",
+       "      <td>0.315</td>\n",
+       "      <td>0.328</td>\n",
+       "      <td>0.154</td>\n",
+       "      <td>0.284</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.368</td>\n",
+       "      <td>0.390</td>\n",
+       "      <td>0.511</td>\n",
+       "      <td>0.498</td>\n",
+       "      <td>0.683</td>\n",
+       "      <td>0.590</td>\n",
+       "      <td>0.3055</td>\n",
+       "      <td>0.3170</td>\n",
+       "      <td>0.245067</td>\n",
+       "      <td>0.261686</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>big-run-refinedweb</td>\n",
+       "      <td>6</td>\n",
+       "      <td>3000</td>\n",
+       "      <td>0.387825</td>\n",
+       "      <td>0.282</td>\n",
+       "      <td>0.287</td>\n",
+       "      <td>0.331</td>\n",
+       "      <td>0.350</td>\n",
+       "      <td>0.152</td>\n",
+       "      <td>0.306</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.376</td>\n",
+       "      <td>0.386</td>\n",
+       "      <td>0.512</td>\n",
+       "      <td>0.495</td>\n",
+       "      <td>0.748</td>\n",
+       "      <td>0.646</td>\n",
+       "      <td>0.3210</td>\n",
+       "      <td>0.3410</td>\n",
+       "      <td>0.250268</td>\n",
+       "      <td>0.266600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>big-run-refinedweb</td>\n",
+       "      <td>6</td>\n",
+       "      <td>4000</td>\n",
+       "      <td>0.398105</td>\n",
+       "      <td>0.310</td>\n",
+       "      <td>0.318</td>\n",
+       "      <td>0.340</td>\n",
+       "      <td>0.389</td>\n",
+       "      <td>0.168</td>\n",
+       "      <td>0.306</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.371</td>\n",
+       "      <td>0.392</td>\n",
+       "      <td>0.513</td>\n",
+       "      <td>0.495</td>\n",
+       "      <td>0.736</td>\n",
+       "      <td>0.634</td>\n",
+       "      <td>0.3305</td>\n",
+       "      <td>0.3425</td>\n",
+       "      <td>0.250732</td>\n",
+       "      <td>0.268341</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1339</th>\n",
+       "      <td>big-run-url_dedups_lowercase_char_length</td>\n",
+       "      <td>6</td>\n",
+       "      <td>163000</td>\n",
+       "      <td>0.477694</td>\n",
+       "      <td>0.396</td>\n",
+       "      <td>0.375</td>\n",
+       "      <td>0.477</td>\n",
+       "      <td>0.578</td>\n",
+       "      <td>0.226</td>\n",
+       "      <td>0.354</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.408</td>\n",
+       "      <td>0.415</td>\n",
+       "      <td>0.562</td>\n",
+       "      <td>0.548</td>\n",
+       "      <td>0.879</td>\n",
+       "      <td>0.817</td>\n",
+       "      <td>0.4655</td>\n",
+       "      <td>0.4540</td>\n",
+       "      <td>0.303672</td>\n",
+       "      <td>0.325554</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1340</th>\n",
+       "      <td>big-run-url_dedups_lowercase_char_length</td>\n",
+       "      <td>6</td>\n",
+       "      <td>164000</td>\n",
+       "      <td>0.476591</td>\n",
+       "      <td>0.396</td>\n",
+       "      <td>0.375</td>\n",
+       "      <td>0.478</td>\n",
+       "      <td>0.581</td>\n",
+       "      <td>0.228</td>\n",
+       "      <td>0.342</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.417</td>\n",
+       "      <td>0.414</td>\n",
+       "      <td>0.555</td>\n",
+       "      <td>0.544</td>\n",
+       "      <td>0.883</td>\n",
+       "      <td>0.827</td>\n",
+       "      <td>0.4600</td>\n",
+       "      <td>0.4570</td>\n",
+       "      <td>0.306406</td>\n",
+       "      <td>0.329724</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1341</th>\n",
+       "      <td>big-run-url_dedups_lowercase_char_length</td>\n",
+       "      <td>6</td>\n",
+       "      <td>165000</td>\n",
+       "      <td>0.478964</td>\n",
+       "      <td>0.405</td>\n",
+       "      <td>0.388</td>\n",
+       "      <td>0.474</td>\n",
+       "      <td>0.583</td>\n",
+       "      <td>0.230</td>\n",
+       "      <td>0.362</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.414</td>\n",
+       "      <td>0.412</td>\n",
+       "      <td>0.562</td>\n",
+       "      <td>0.541</td>\n",
+       "      <td>0.881</td>\n",
+       "      <td>0.826</td>\n",
+       "      <td>0.4545</td>\n",
+       "      <td>0.4465</td>\n",
+       "      <td>0.304121</td>\n",
+       "      <td>0.327213</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1342</th>\n",
+       "      <td>big-run-url_dedups_lowercase_char_length</td>\n",
+       "      <td>6</td>\n",
+       "      <td>166000</td>\n",
+       "      <td>0.477467</td>\n",
+       "      <td>0.398</td>\n",
+       "      <td>0.381</td>\n",
+       "      <td>0.470</td>\n",
+       "      <td>0.579</td>\n",
+       "      <td>0.234</td>\n",
+       "      <td>0.354</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.413</td>\n",
+       "      <td>0.411</td>\n",
+       "      <td>0.554</td>\n",
+       "      <td>0.544</td>\n",
+       "      <td>0.887</td>\n",
+       "      <td>0.831</td>\n",
+       "      <td>0.4625</td>\n",
+       "      <td>0.4565</td>\n",
+       "      <td>0.305855</td>\n",
+       "      <td>0.328240</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1343</th>\n",
+       "      <td>big-run-url_dedups_lowercase_char_length</td>\n",
+       "      <td>6</td>\n",
+       "      <td>167000</td>\n",
+       "      <td>0.476630</td>\n",
+       "      <td>0.398</td>\n",
+       "      <td>0.370</td>\n",
+       "      <td>0.477</td>\n",
+       "      <td>0.577</td>\n",
+       "      <td>0.244</td>\n",
+       "      <td>0.354</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.413</td>\n",
+       "      <td>0.414</td>\n",
+       "      <td>0.553</td>\n",
+       "      <td>0.540</td>\n",
+       "      <td>0.879</td>\n",
+       "      <td>0.825</td>\n",
+       "      <td>0.4660</td>\n",
+       "      <td>0.4565</td>\n",
+       "      <td>0.307940</td>\n",
+       "      <td>0.328538</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>1344 rows × 22 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                       runname  seed   steps  agg_score  \\\n",
+       "0                           big-run-refinedweb     6       0   0.330893   \n",
+       "1                           big-run-refinedweb     6    1000   0.353481   \n",
+       "2                           big-run-refinedweb     6    2000   0.376461   \n",
+       "3                           big-run-refinedweb     6    3000   0.387825   \n",
+       "4                           big-run-refinedweb     6    4000   0.398105   \n",
+       "...                                        ...   ...     ...        ...   \n",
+       "1339  big-run-url_dedups_lowercase_char_length     6  163000   0.477694   \n",
+       "1340  big-run-url_dedups_lowercase_char_length     6  164000   0.476591   \n",
+       "1341  big-run-url_dedups_lowercase_char_length     6  165000   0.478964   \n",
+       "1342  big-run-url_dedups_lowercase_char_length     6  166000   0.477467   \n",
+       "1343  big-run-url_dedups_lowercase_char_length     6  167000   0.476630   \n",
+       "\n",
+       "      commonsense_qa/acc  commonsense_qa/acc_norm  hellaswag/acc  \\\n",
+       "0                  0.186                    0.233          0.272   \n",
+       "1                  0.233                    0.253          0.288   \n",
+       "2                  0.282                    0.280          0.315   \n",
+       "3                  0.282                    0.287          0.331   \n",
+       "4                  0.310                    0.318          0.340   \n",
+       "...                  ...                      ...            ...   \n",
+       "1339               0.396                    0.375          0.477   \n",
+       "1340               0.396                    0.375          0.478   \n",
+       "1341               0.405                    0.388          0.474   \n",
+       "1342               0.398                    0.381          0.470   \n",
+       "1343               0.398                    0.370          0.477   \n",
+       "\n",
+       "      hellaswag/acc_norm  openbookqa/acc  openbookqa/acc_norm  ...  siqa/acc  \\\n",
+       "0                  0.258           0.166                0.286  ...     0.367   \n",
+       "1                  0.276           0.120                0.256  ...     0.365   \n",
+       "2                  0.328           0.154                0.284  ...     0.368   \n",
+       "3                  0.350           0.152                0.306  ...     0.376   \n",
+       "4                  0.389           0.168                0.306  ...     0.371   \n",
+       "...                  ...             ...                  ...  ...       ...   \n",
+       "1339               0.578           0.226                0.354  ...     0.408   \n",
+       "1340               0.581           0.228                0.342  ...     0.417   \n",
+       "1341               0.583           0.230                0.362  ...     0.414   \n",
+       "1342               0.579           0.234                0.354  ...     0.413   \n",
+       "1343               0.577           0.244                0.354  ...     0.413   \n",
+       "\n",
+       "      siqa/acc_norm  winogrande/acc  winogrande/acc_norm  sciq/acc  \\\n",
+       "0             0.362           0.516                0.497     0.208   \n",
+       "1             0.398           0.502                0.500     0.582   \n",
+       "2             0.390           0.511                0.498     0.683   \n",
+       "3             0.386           0.512                0.495     0.748   \n",
+       "4             0.392           0.513                0.495     0.736   \n",
+       "...             ...             ...                  ...       ...   \n",
+       "1339          0.415           0.562                0.548     0.879   \n",
+       "1340          0.414           0.555                0.544     0.883   \n",
+       "1341          0.412           0.562                0.541     0.881   \n",
+       "1342          0.411           0.554                0.544     0.887   \n",
+       "1343          0.414           0.553                0.540     0.879   \n",
+       "\n",
+       "      sciq/acc_norm  arc/acc  arc/acc_norm  mmlu/acc  mmlu/acc_norm  \n",
+       "0             0.202   0.2195        0.2510  0.230294       0.250147  \n",
+       "1             0.528   0.2650        0.2900  0.240583       0.252852  \n",
+       "2             0.590   0.3055        0.3170  0.245067       0.261686  \n",
+       "3             0.646   0.3210        0.3410  0.250268       0.266600  \n",
+       "4             0.634   0.3305        0.3425  0.250732       0.268341  \n",
+       "...             ...      ...           ...       ...            ...  \n",
+       "1339          0.817   0.4655        0.4540  0.303672       0.325554  \n",
+       "1340          0.827   0.4600        0.4570  0.306406       0.329724  \n",
+       "1341          0.826   0.4545        0.4465  0.304121       0.327213  \n",
+       "1342          0.831   0.4625        0.4565  0.305855       0.328240  \n",
+       "1343          0.825   0.4660        0.4565  0.307940       0.328538  \n",
+       "\n",
+       "[1344 rows x 22 columns]"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "from matplotlib.figure import Figure\n",
+    "\n",
+    "df = pd.read_csv(\"../src_data/diff_dedup_attempts.csv\")\n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "874ab88a573cd443",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-13T13:56:19.453420Z",
+     "start_time": "2024-05-13T13:56:19.450850Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['big-run-refinedweb',\n",
+       " 'big-run-sampled_cross_minhash_dump',\n",
+       " 'big-run-sampled_full_filtered_no_dedup',\n",
+       " 'big-run-sampled_full_imh_linededup',\n",
+       " 'big-run-sampled_full_ind_minhash',\n",
+       " 'big-run-sampled_line_dedup_3lines2',\n",
+       " 'big-run-sampled_line_dedup_min_words',\n",
+       " 'big-run-url_dedups_lowercase_char_length']"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "pd.unique(df[\"runname\"]).tolist()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "b610f43caefdf01",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-13T14:00:46.578560Z",
+     "start_time": "2024-05-13T14:00:46.576167Z"
+    },
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "runs_mapping = {\n",
+    "    \"big-run-refinedweb\": \"RefinedWeb\",\n",
+    "    \"big-run-sampled_cross_minhash_dump\": \"FineWeb full MinHash\",\n",
+    "    \"big-run-sampled_full_filtered_no_dedup\": \"FineWeb filtered only\",\n",
+    "    \"big-run-sampled_full_ind_minhash\": \"FineWeb independent MinHash\",\n",
+    "    \"big-run-sampled_full_imh_linededup\": \"FineWeb line dedup\",\n",
+    "    \"big-run-sampled_line_dedup_3lines2\": \"FineWeb 3-line dedup\",\n",
+    "    \"big-run-sampled_line_dedup_min_words\": \"FineWeb line dedup w/ min words\",\n",
+    "    \"big-run-url_dedups_lowercase_char_length\": \"FineWeb URL dedup\"\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "initial_id",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-13T14:04:41.777032Z",
+     "start_time": "2024-05-13T14:04:41.536919Z"
+    },
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "import os\n",
+    "from matplotlib import pyplot as plt\n",
+    "metrics = ['agg_score', 'commonsense_qa/acc_norm', 'hellaswag/acc_norm', 'openbookqa/acc_norm', 'piqa/acc_norm',\n",
+    "                   'siqa/acc_norm', 'winogrande/acc_norm', 'arc/acc_norm', 'mmlu/acc_norm']\n",
+    "\n",
+    "def normalize_runname(runname):\n",
+    "    return runname.replace(\"/\", \"_\")\n",
+    "\n",
+    "grouped = (\n",
+    "    df.groupby([\"runname\", \"steps\"])\n",
+    "    .agg(\n",
+    "        {\n",
+    "            key: \"mean\" for key in metrics\n",
+    "        }\n",
+    "    )\n",
+    "    .reset_index()\n",
+    ")\n",
+    "\n",
+    "file_id=\"../assets/data/plots/dedup_attempts\"\n",
+    "files = {}\n",
+    "for metric in metrics:\n",
+    "    datas = {}\n",
+    "    for name, group in grouped.groupby(\"runname\"):\n",
+    "        group = group[[\"steps\", metric]].sort_values(by=\"steps\")\n",
+    "        group = group.set_index(\"steps\")\n",
+    "        rolling_avg = group\n",
+    "        # rolling_avg = group.rolling(wjjjjjjjjjjjjjindow=5).mean()\n",
+    "        datas[name] = {\n",
+    "            \"x\": (rolling_avg.index * 2048 * 1024 * 1e-9).tolist(),\n",
+    "            \"y\": rolling_avg[metric].tolist(),\n",
+    "            \"label\": runs_mapping[name],\n",
+    "        }\n",
+    "    # Sort the datata based on the steps\n",
+    "    datas = {k: v for k, v in sorted(datas.items(), key=lambda x: -x[1][\"y\"][-1])}\n",
+    "    # Create a folder\n",
+    "    os.makedirs(f\"{file_id}\", exist_ok=True)\n",
+    "    with open(f\"{file_id}/{normalize_runname(metric)}.json\", \"w\") as f:\n",
+    "        json.dump({\n",
+    "            \"data\": datas,\n",
+    "            \"layout\": {\n",
+    "                \"title\": {\n",
+    "                    \"text\": \"Attempting to further globally dedup worsened perf\"\n",
+    "                },\n",
+    "            }\n",
+    "        }, f)\n",
+    "    files[metric] = {\"file\": f\"{normalize_runname(metric)}.json\"}\n",
+    "# Create index\n",
+    "with open(f\"{file_id}/index.json\", \"w\") as f:\n",
+    "    json.dump({\n",
+    "        \"files\": files,\n",
+    "        \"settings\": {\n",
+    "            \"defaultMetric\": \"agg_score\",\n",
+    "            \"slider\":{\"min\":0,\"max\":30,\"default\":5}\n",
+    "        }\n",
+    "    }, f)\n",
+    "        \n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "af28ebbd054cdc33",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-04-30T15:08:02.522543Z",
+     "start_time": "2024-04-30T15:08:02.520569Z"
+    },
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/plot_dedup_ind_dedup_better.ipynb

@@ -0,0 +1,570 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "138889b92720ce2e",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-04-30T15:08:02.398435Z",
+     "start_time": "2024-04-30T15:08:02.194901Z"
+    },
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>runname</th>\n",
+       "      <th>seed</th>\n",
+       "      <th>steps</th>\n",
+       "      <th>agg_score</th>\n",
+       "      <th>commonsense_qa/acc</th>\n",
+       "      <th>commonsense_qa/acc_norm</th>\n",
+       "      <th>hellaswag/acc</th>\n",
+       "      <th>hellaswag/acc_norm</th>\n",
+       "      <th>openbookqa/acc</th>\n",
+       "      <th>openbookqa/acc_norm</th>\n",
+       "      <th>...</th>\n",
+       "      <th>siqa/acc</th>\n",
+       "      <th>siqa/acc_norm</th>\n",
+       "      <th>winogrande/acc</th>\n",
+       "      <th>winogrande/acc_norm</th>\n",
+       "      <th>sciq/acc</th>\n",
+       "      <th>sciq/acc_norm</th>\n",
+       "      <th>arc/acc</th>\n",
+       "      <th>arc/acc_norm</th>\n",
+       "      <th>mmlu/acc</th>\n",
+       "      <th>mmlu/acc_norm</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>big-run-sampled_full_filtered_no_dedup</td>\n",
+       "      <td>6</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.330893</td>\n",
+       "      <td>0.186</td>\n",
+       "      <td>0.233</td>\n",
+       "      <td>0.272</td>\n",
+       "      <td>0.258</td>\n",
+       "      <td>0.166</td>\n",
+       "      <td>0.286</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.367</td>\n",
+       "      <td>0.362</td>\n",
+       "      <td>0.516</td>\n",
+       "      <td>0.497</td>\n",
+       "      <td>0.209</td>\n",
+       "      <td>0.202</td>\n",
+       "      <td>0.2195</td>\n",
+       "      <td>0.2510</td>\n",
+       "      <td>0.230294</td>\n",
+       "      <td>0.250147</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>big-run-sampled_full_filtered_no_dedup</td>\n",
+       "      <td>6</td>\n",
+       "      <td>1000</td>\n",
+       "      <td>0.360520</td>\n",
+       "      <td>0.254</td>\n",
+       "      <td>0.260</td>\n",
+       "      <td>0.290</td>\n",
+       "      <td>0.281</td>\n",
+       "      <td>0.138</td>\n",
+       "      <td>0.256</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.362</td>\n",
+       "      <td>0.400</td>\n",
+       "      <td>0.517</td>\n",
+       "      <td>0.524</td>\n",
+       "      <td>0.573</td>\n",
+       "      <td>0.515</td>\n",
+       "      <td>0.2675</td>\n",
+       "      <td>0.2895</td>\n",
+       "      <td>0.239489</td>\n",
+       "      <td>0.251660</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>big-run-sampled_full_filtered_no_dedup</td>\n",
+       "      <td>6</td>\n",
+       "      <td>2000</td>\n",
+       "      <td>0.373315</td>\n",
+       "      <td>0.285</td>\n",
+       "      <td>0.278</td>\n",
+       "      <td>0.315</td>\n",
+       "      <td>0.323</td>\n",
+       "      <td>0.138</td>\n",
+       "      <td>0.272</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.365</td>\n",
+       "      <td>0.395</td>\n",
+       "      <td>0.509</td>\n",
+       "      <td>0.490</td>\n",
+       "      <td>0.677</td>\n",
+       "      <td>0.596</td>\n",
+       "      <td>0.3075</td>\n",
+       "      <td>0.3235</td>\n",
+       "      <td>0.250318</td>\n",
+       "      <td>0.261019</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>big-run-sampled_full_filtered_no_dedup</td>\n",
+       "      <td>6</td>\n",
+       "      <td>3000</td>\n",
+       "      <td>0.388201</td>\n",
+       "      <td>0.294</td>\n",
+       "      <td>0.291</td>\n",
+       "      <td>0.327</td>\n",
+       "      <td>0.341</td>\n",
+       "      <td>0.152</td>\n",
+       "      <td>0.298</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.371</td>\n",
+       "      <td>0.396</td>\n",
+       "      <td>0.512</td>\n",
+       "      <td>0.504</td>\n",
+       "      <td>0.712</td>\n",
+       "      <td>0.621</td>\n",
+       "      <td>0.3220</td>\n",
+       "      <td>0.3390</td>\n",
+       "      <td>0.255646</td>\n",
+       "      <td>0.266605</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>big-run-sampled_full_filtered_no_dedup</td>\n",
+       "      <td>6</td>\n",
+       "      <td>4000</td>\n",
+       "      <td>0.393412</td>\n",
+       "      <td>0.306</td>\n",
+       "      <td>0.307</td>\n",
+       "      <td>0.337</td>\n",
+       "      <td>0.360</td>\n",
+       "      <td>0.172</td>\n",
+       "      <td>0.284</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.380</td>\n",
+       "      <td>0.402</td>\n",
+       "      <td>0.522</td>\n",
+       "      <td>0.510</td>\n",
+       "      <td>0.729</td>\n",
+       "      <td>0.612</td>\n",
+       "      <td>0.3100</td>\n",
+       "      <td>0.3385</td>\n",
+       "      <td>0.253048</td>\n",
+       "      <td>0.266798</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>670</th>\n",
+       "      <td>big-run-sampled_full_ind_minhash</td>\n",
+       "      <td>6</td>\n",
+       "      <td>163000</td>\n",
+       "      <td>0.481842</td>\n",
+       "      <td>0.427</td>\n",
+       "      <td>0.393</td>\n",
+       "      <td>0.488</td>\n",
+       "      <td>0.579</td>\n",
+       "      <td>0.242</td>\n",
+       "      <td>0.358</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.420</td>\n",
+       "      <td>0.397</td>\n",
+       "      <td>0.587</td>\n",
+       "      <td>0.568</td>\n",
+       "      <td>0.885</td>\n",
+       "      <td>0.809</td>\n",
+       "      <td>0.4760</td>\n",
+       "      <td>0.4595</td>\n",
+       "      <td>0.305843</td>\n",
+       "      <td>0.330238</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>671</th>\n",
+       "      <td>big-run-sampled_full_ind_minhash</td>\n",
+       "      <td>6</td>\n",
+       "      <td>164000</td>\n",
+       "      <td>0.482727</td>\n",
+       "      <td>0.426</td>\n",
+       "      <td>0.394</td>\n",
+       "      <td>0.487</td>\n",
+       "      <td>0.582</td>\n",
+       "      <td>0.238</td>\n",
+       "      <td>0.360</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.422</td>\n",
+       "      <td>0.398</td>\n",
+       "      <td>0.575</td>\n",
+       "      <td>0.562</td>\n",
+       "      <td>0.885</td>\n",
+       "      <td>0.827</td>\n",
+       "      <td>0.4745</td>\n",
+       "      <td>0.4625</td>\n",
+       "      <td>0.307377</td>\n",
+       "      <td>0.332317</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>672</th>\n",
+       "      <td>big-run-sampled_full_ind_minhash</td>\n",
+       "      <td>6</td>\n",
+       "      <td>165000</td>\n",
+       "      <td>0.482413</td>\n",
+       "      <td>0.423</td>\n",
+       "      <td>0.397</td>\n",
+       "      <td>0.482</td>\n",
+       "      <td>0.573</td>\n",
+       "      <td>0.238</td>\n",
+       "      <td>0.360</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.409</td>\n",
+       "      <td>0.396</td>\n",
+       "      <td>0.581</td>\n",
+       "      <td>0.569</td>\n",
+       "      <td>0.889</td>\n",
+       "      <td>0.829</td>\n",
+       "      <td>0.4675</td>\n",
+       "      <td>0.4600</td>\n",
+       "      <td>0.308059</td>\n",
+       "      <td>0.331304</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>673</th>\n",
+       "      <td>big-run-sampled_full_ind_minhash</td>\n",
+       "      <td>6</td>\n",
+       "      <td>166000</td>\n",
+       "      <td>0.482014</td>\n",
+       "      <td>0.422</td>\n",
+       "      <td>0.391</td>\n",
+       "      <td>0.477</td>\n",
+       "      <td>0.573</td>\n",
+       "      <td>0.230</td>\n",
+       "      <td>0.358</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.420</td>\n",
+       "      <td>0.400</td>\n",
+       "      <td>0.586</td>\n",
+       "      <td>0.566</td>\n",
+       "      <td>0.883</td>\n",
+       "      <td>0.817</td>\n",
+       "      <td>0.4660</td>\n",
+       "      <td>0.4645</td>\n",
+       "      <td>0.304975</td>\n",
+       "      <td>0.329611</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>674</th>\n",
+       "      <td>big-run-sampled_full_ind_minhash</td>\n",
+       "      <td>6</td>\n",
+       "      <td>167000</td>\n",
+       "      <td>0.486587</td>\n",
+       "      <td>0.424</td>\n",
+       "      <td>0.402</td>\n",
+       "      <td>0.490</td>\n",
+       "      <td>0.579</td>\n",
+       "      <td>0.236</td>\n",
+       "      <td>0.360</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.417</td>\n",
+       "      <td>0.405</td>\n",
+       "      <td>0.585</td>\n",
+       "      <td>0.575</td>\n",
+       "      <td>0.884</td>\n",
+       "      <td>0.832</td>\n",
+       "      <td>0.4760</td>\n",
+       "      <td>0.4715</td>\n",
+       "      <td>0.309503</td>\n",
+       "      <td>0.332197</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>675 rows × 22 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                    runname  seed   steps  agg_score  \\\n",
+       "0    big-run-sampled_full_filtered_no_dedup     6       0   0.330893   \n",
+       "1    big-run-sampled_full_filtered_no_dedup     6    1000   0.360520   \n",
+       "2    big-run-sampled_full_filtered_no_dedup     6    2000   0.373315   \n",
+       "3    big-run-sampled_full_filtered_no_dedup     6    3000   0.388201   \n",
+       "4    big-run-sampled_full_filtered_no_dedup     6    4000   0.393412   \n",
+       "..                                      ...   ...     ...        ...   \n",
+       "670        big-run-sampled_full_ind_minhash     6  163000   0.481842   \n",
+       "671        big-run-sampled_full_ind_minhash     6  164000   0.482727   \n",
+       "672        big-run-sampled_full_ind_minhash     6  165000   0.482413   \n",
+       "673        big-run-sampled_full_ind_minhash     6  166000   0.482014   \n",
+       "674        big-run-sampled_full_ind_minhash     6  167000   0.486587   \n",
+       "\n",
+       "     commonsense_qa/acc  commonsense_qa/acc_norm  hellaswag/acc  \\\n",
+       "0                 0.186                    0.233          0.272   \n",
+       "1                 0.254                    0.260          0.290   \n",
+       "2                 0.285                    0.278          0.315   \n",
+       "3                 0.294                    0.291          0.327   \n",
+       "4                 0.306                    0.307          0.337   \n",
+       "..                  ...                      ...            ...   \n",
+       "670               0.427                    0.393          0.488   \n",
+       "671               0.426                    0.394          0.487   \n",
+       "672               0.423                    0.397          0.482   \n",
+       "673               0.422                    0.391          0.477   \n",
+       "674               0.424                    0.402          0.490   \n",
+       "\n",
+       "     hellaswag/acc_norm  openbookqa/acc  openbookqa/acc_norm  ...  siqa/acc  \\\n",
+       "0                 0.258           0.166                0.286  ...     0.367   \n",
+       "1                 0.281           0.138                0.256  ...     0.362   \n",
+       "2                 0.323           0.138                0.272  ...     0.365   \n",
+       "3                 0.341           0.152                0.298  ...     0.371   \n",
+       "4                 0.360           0.172                0.284  ...     0.380   \n",
+       "..                  ...             ...                  ...  ...       ...   \n",
+       "670               0.579           0.242                0.358  ...     0.420   \n",
+       "671               0.582           0.238                0.360  ...     0.422   \n",
+       "672               0.573           0.238                0.360  ...     0.409   \n",
+       "673               0.573           0.230                0.358  ...     0.420   \n",
+       "674               0.579           0.236                0.360  ...     0.417   \n",
+       "\n",
+       "     siqa/acc_norm  winogrande/acc  winogrande/acc_norm  sciq/acc  \\\n",
+       "0            0.362           0.516                0.497     0.209   \n",
+       "1            0.400           0.517                0.524     0.573   \n",
+       "2            0.395           0.509                0.490     0.677   \n",
+       "3            0.396           0.512                0.504     0.712   \n",
+       "4            0.402           0.522                0.510     0.729   \n",
+       "..             ...             ...                  ...       ...   \n",
+       "670          0.397           0.587                0.568     0.885   \n",
+       "671          0.398           0.575                0.562     0.885   \n",
+       "672          0.396           0.581                0.569     0.889   \n",
+       "673          0.400           0.586                0.566     0.883   \n",
+       "674          0.405           0.585                0.575     0.884   \n",
+       "\n",
+       "     sciq/acc_norm  arc/acc  arc/acc_norm  mmlu/acc  mmlu/acc_norm  \n",
+       "0            0.202   0.2195        0.2510  0.230294       0.250147  \n",
+       "1            0.515   0.2675        0.2895  0.239489       0.251660  \n",
+       "2            0.596   0.3075        0.3235  0.250318       0.261019  \n",
+       "3            0.621   0.3220        0.3390  0.255646       0.266605  \n",
+       "4            0.612   0.3100        0.3385  0.253048       0.266798  \n",
+       "..             ...      ...           ...       ...            ...  \n",
+       "670          0.809   0.4760        0.4595  0.305843       0.330238  \n",
+       "671          0.827   0.4745        0.4625  0.307377       0.332317  \n",
+       "672          0.829   0.4675        0.4600  0.308059       0.331304  \n",
+       "673          0.817   0.4660        0.4645  0.304975       0.329611  \n",
+       "674          0.832   0.4760        0.4715  0.309503       0.332197  \n",
+       "\n",
+       "[675 rows x 22 columns]"
+      ]
+     },
+     "execution_count": 19,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "from matplotlib.figure import Figure\n",
+    "\n",
+    "df = pd.read_csv(\"../src_data/cross_ind_unfiltered_comparison.csv\")\n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "b610f43caefdf01",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-04-30T15:08:02.401852Z",
+     "start_time": "2024-04-30T15:08:02.399712Z"
+    },
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "runs_mapping = {\n",
+    "    \"big-run-refinedweb\": \"RefinedWeb\",\n",
+    "    \"big-run-fineweb-cross-dedup-fixed\": \"FineWeb full MinHash\",\n",
+    "    \"big-run-sampled_full_ind_minhash\": \"FineWeb independent MinHash\",\n",
+    "    \"big-run-sampled_full_filtered_no_dedup\": \"FineWeb filtered only\"\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "initial_id",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-04-30T15:08:02.519228Z",
+     "start_time": "2024-04-30T15:08:02.402938Z"
+    },
+    "collapsed": true
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
+     ]
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkIAAAHHCAYAAABTMjf2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAA9hAAAPYQGoP6dpAABLBElEQVR4nO3deXRN5/7H8U8SMpEBkQEhhCo13qg0hipSKUr1Vg1VCS1tTVWp25pDq6KKq9fMbVHtLar0tjVW8GspNWtras3aSsyJOZI8vz+6cm6PJCRkEPv9Wuus5Tx59t7ffZ59zvnY03EwxhgBAABYkGNBFwAAAFBQCEIAAMCyCEIAAMCyCEIAAMCyCEIAAMCyCEIAAMCyCEIAAMCyCEIAAMCyCEIAAMCyCEKFyNGjR+Xg4KC5c+cWdCm5ojCtT17UOnfuXDk4OOjo0aO5Nk/86b333lOlSpXk5OSkOnXqFHQ5ecbBwUEjR47MtfmtX79eDg4OWr9+fa7NE9kXFBSkbt26FXQZlkMQyiPpX3Lbtm0r6FIsb9q0aYUibBV2Y8aM0RdffFHQZWj16tV644031LBhQ82ZM0djxowp6JIA3MOKFHQBQF6bNm2afHx8+J9WHhszZozat2+vdu3aFWgda9eulaOjoz744AM5OzsXaC0A7n3sEQJwX7hy5Yok6dSpU3Jzc8u1EGSM0dWrV3NlXig4165dU1paWkGXgXsQQSgfdevWTcWLF9fvv/+udu3aqXjx4ipdurQGDhyo1NRUu74XLlxQt27d5OXlJW9vb0VFRenChQuZznf//v1q3769SpYsKVdXV9WrV09ffvmlXZ/0Q3XffvutXn75ZZUqVUqenp6KjIzU+fPnM8xzxYoVaty4sYoVKyYPDw+1bt1ae/bsuefWZ+PGjYqOjlbp0qVVrFgxPf300zp9+rStX1BQkPbs2aP/+7//k4ODgxwcHPTYY49luty8qlWS9uzZo2bNmsnNzU3lypXT6NGjM/1Qzuqcj5vPHcjpeGZm7dq1tjH29vbWU089pX379tn16datm4KCgjJMO3LkSDk4ONjVffnyZc2bN8/2OqfXm953//796tChgzw9PVWqVCn1799f165dyzDvjz/+WCEhIXJzc1PJkiXVqVMnnThxwq7PY489pho1amj79u169NFH5e7uriFDhsjBwUFz5szR5cuXbXWkHxZNSUnR22+/reDgYLm4uCgoKEhDhgzR9evXM7zWTz75pFatWqV69erJzc1NM2fOtJ0/s2jRIo0aNUply5aVh4eH2rdvr8TERF2/fl2vvfaafH19Vbx4cXXv3j3DvL/55hs1atRI3t7eKl68uKpWraohQ4bcdqyuX7+uAQMGqHTp0vLw8FDbtm3122+/Zdr3999/1wsvvCA/Pz+5uLjooYce0ocffpih32+//aZ27dqpWLFi8vX11YABAzLUm/56ZLY39bHHHrN7L6W/PgsXLtSQIUPk7++vYsWKqW3bthnGLzPHjh1T7969VbVqVbm5ualUqVJ69tlnMz2H7sKFCxowYICCgoLk4uKicuXKKTIyUmfOnLGrZcGCBRo2bJjKli0rd3d3JSUlSZI+++wz2zbm4+Oj559/Xr///rvdMuLj49W9e3eVK1dOLi4uCggI0FNPPWVXz7Zt2xQRESEfHx+5ubmpYsWKeuGFF267rsYYjR49WuXKlZO7u7uaNm2a4fNVyvg+S5fZ+YXp2+369ett223NmjVt53stWbJENWvWlKurq0JCQrRz5067eaZ/nh8+fFgREREqVqyYypQpo7feekvGGLu+CxYsUEhIiDw8POTp6amaNWvq/fffv+1636s4NJbPUlNTFRERodDQUI0fP15r1qzRhAkTFBwcrF69ekn6803y1FNPacOGDXrllVdUrVo1LV26VFFRURnmt2fPHjVs2FBly5bVoEGDVKxYMS1atEjt2rXT559/rqefftquf9++feXt7a2RI0fqwIEDmj59uo4dO2b74JCk+fPnKyoqShEREXr33Xd15coVTZ8+XY0aNdLOnTvtvhgLen369eunEiVKKCYmRkePHtWkSZPUt29fLVy4UJI0adIk9evXT8WLF9fQoUMlSX5+flmOT17UGh8fr6ZNmyolJcXWb9asWXJzc8uyjuzKznhmZs2aNWrZsqUqVaqkkSNH6urVq5o8ebIaNmyoHTt2ZBp+bmX+/Pnq0aOH6tevr5deekmSFBwcbNenQ4cOCgoKUmxsrDZv3qx//etfOn/+vD766CNbn3feeUfDhw9Xhw4d1KNHD50+fVqTJ0/Wo48+qp07d8rb29vW9+zZs2rZsqU6deqk559/Xn5+fqpXr55mzZqlLVu26N///rckqUGDBpKkHj16aN68eWrfvr1ef/11/fDDD4qNjdW+ffu0dOlSu1oPHDigzp076+WXX1bPnj1VtWpV299iY2Pl5uamQYMG6eDBg5o8ebKKFi0qR0dHnT9/XiNHjtTmzZs1d+5cVaxYUSNGjJD05/by5JNPqlatWnrrrbfk4uKigwcPauPGjbd9fXv06KGPP/5Yzz33nBo0aKC1a9eqdevWGfolJCTokUcekYODg/r27avSpUtrxYoVevHFF5WUlKTXXntNknT16lU1b95cx48f16uvvqoyZcpo/vz5Wrt27W1ruZ133nlHDg4OevPNN3Xq1ClNmjRJ4eHh2rVr1y23+a1bt+r7779Xp06dVK5cOR09elTTp0/XY489pr1798rd3V2SdOnSJTVu3Fj79u3TCy+8oL/97W86c+aMvvzyS/3222/y8fGxzfPtt9+Ws7OzBg4cqOvXr8vZ2Vlz585V9+7d9fDDDys2NlYJCQl6//33tXHjRrtt7JlnntGePXvUr18/BQUF6dSpU/rmm290/Phx2/MWLVqodOnSGjRokLy9vXX06FEtWbLktq/RiBEjNHr0aLVq1UqtWrXSjh071KJFCyUnJ9/Va3/w4EE999xzevnll/X8889r/PjxatOmjWbMmKEhQ4aod+/ekv7chjt06KADBw7I0fF/+0NSU1P1xBNP6JFHHtG4ceO0cuVKxcTEKCUlRW+99ZakP8N8586d1bx5c7377ruSpH379mnjxo3q37//XdVfYAzyxJw5c4wks3XrVltbVFSUkWTeeustu75169Y1ISEhtudffPGFkWTGjRtna0tJSTGNGzc2ksycOXNs7c2bNzc1a9Y0165ds7WlpaWZBg0amCpVqmSoJyQkxCQnJ9vax40bZySZ//73v8YYYy5evGi8vb1Nz5497WqMj483Xl5edu33wvqEh4ebtLQ0W/uAAQOMk5OTuXDhgq3toYceMk2aNDHZkRe1vvbaa0aS+eGHH2xtp06dMl5eXkaSOXLkiK1dkomJiclQV4UKFUxUVFSG9b/deGalTp06xtfX15w9e9bWtnv3buPo6GgiIyNtbVFRUaZChQoZpo+JiTE3f3wUK1bMrsab+7Zt29auvXfv3kaS2b17tzHGmKNHjxonJyfzzjvv2PX76aefTJEiRezamzRpYiSZGTNmZFheVFSUKVasmF3brl27jCTTo0cPu/aBAwcaSWbt2rW2tgoVKhhJZuXKlXZ9161bZySZGjVq2L3mnTt3Ng4ODqZly5Z2/cPCwuxeu3/+859Gkjl9+nSGmm8lvfbevXvbtT/33HMZtpcXX3zRBAQEmDNnztj17dSpk/Hy8jJXrlwxxhgzadIkI8ksWrTI1ufy5cumcuXKRpJZt26d3euR2bg2adLE7n2V/vqULVvWJCUl2doXLVpkJJn333//luuZXttfbdq0yUgyH330ka1txIgRRpJZsmRJhv7pnwXptVSqVMluvsnJycbX19fUqFHDXL161db+9ddfG0lmxIgRxhhjzp8/bySZ9957L8t6ly5dmuEzPjtOnTplnJ2dTevWre0+u4YMGWIk2b3Wmb3PjPnf+/+vnx3p2+33339va1u1apWRZNzc3MyxY8ds7TNnzswwzumf5/369bO1paWlmdatWxtnZ2fbdtu/f3/j6elpUlJScrTe9zIOjRWAV155xe5548aNdfjwYdvz5cuXq0iRIrY9KpLk5OSkfv362U137tw5rV27Vh06dNDFixd15swZnTlzRmfPnlVERIR+/fXXDLt7X3rpJRUtWtT2vFevXipSpIiWL18u6c+0f+HCBXXu3Nk2vzNnzsjJyUmhoaFat27dPbc+f93z0bhxY6WmpurYsWMZ6syOvKh1+fLleuSRR1S/fn3b9KVLl1aXLl3uqMa/ut14ZubkyZPatWuXunXrppIlS9raa9Wqpccff/yW096NPn362D1Pf03Tl7dkyRKlpaWpQ4cOdtuev7+/qlSpkmHbc3FxUffu3bO17PRlREdH27W//vrrkqRly5bZtVesWFERERGZzisyMtLuNQ8NDZUxJsMhkdDQUJ04cUIpKSmSZNvT8N///jdH56qk1/7qq6/atafv3UlnjNHnn3+uNm3ayBhj9xpGREQoMTFRO3bssM0zICBA7du3t03v7u5u25t3NyIjI+Xh4WF73r59ewUEBNx2u/rr3qIbN27o7Nmzqly5sry9vW11S9Lnn3+u2rVrZ9g7LCnDXtCoqCi7+W7btk2nTp1S79695erqamtv3bq1HnzwQdt2kH6O2fr167M81Jw+nl9//bVu3Lhxy3X7qzVr1ig5OVn9+vWzq/fm8bwT1atXV1hYmO15aGioJKlZs2YqX758hva/fk6n69u3r+3f6XsWk5OTtWbNGkl/rvfly5f1zTff3HW99wqCUD5zdXVV6dKl7dpKlChh92Y7duyYAgICVLx4cbt+f909L/25G9QYo+HDh6t06dJ2j5iYGEl/njj6V1WqVLF7Xrx4cQUEBNiONf/666+S/nzj3DzP1atXZ5hfQa/PX9/c6cuWlO3zZG6WF7UeO3Ysw+ue2TzvxO3GMzPpITGz5VerVk1nzpzR5cuX77q2m91ca3BwsBwdHe22PWOMqlSpkuE13bdvX4axL1u2bLZPiD527JgcHR1VuXJlu3Z/f395e3tnCM4VK1bMcl43b3NeXl6SpMDAwAztaWlpSkxMlCR17NhRDRs2VI8ePeTn56dOnTpp0aJFtw1F6bXffKjx5vE7ffq0Lly4oFmzZmV4/dID41+3ycqVK2cIDnmxTTo4OKhy5cq3vV/W1atXNWLECAUGBsrFxUU+Pj4qXbq0Lly4YHsNJenQoUOqUaNGtmq5eRxvte0/+OCDtr+7uLjo3Xff1YoVK+Tn56dHH31U48aNU3x8vK1/kyZN9Mwzz2jUqFHy8fHRU089pTlz5mR6nlVmNdz8OpUuXdr2+XWncrJtShk/Jx0dHVWpUiW7tgceeECSbOPXu3dvPfDAA2rZsqXKlSunF154QStXrryrugsa5wjlMycnp1ybV/oH6MCBA7P83+vNH/zZnef8+fPl7++f4e9FithvMgW9Plkt39x0cl9uy4vX/lZuPvk8v2R1nlFu1HPzvNPS0uTg4KAVK1ZkOq43h9M7OcfqVudNZXfeWW1zt9sW3dzc9O2332rdunVatmyZVq5cqYULF6pZs2ZavXr1Xb+X0rfJ559/PtNz2qQ/9/rl1K22gdx8//fr109z5szRa6+9prCwMHl5ecnBwUGdOnW646u97uY8vNdee01t2rTRF198oVWrVmn48OGKjY3V2rVrVbduXTk4OGjx4sXavHmzvvrqK61atUovvPCCJkyYoM2bN2fYXu9ETt9/d7pt5oSvr6927dqlVatWacWKFVqxYoXmzJmjyMhIzZs3L8fzuxcQhO5BFSpUUFxcnC5dumT3Zjpw4IBdv/TkXrRoUYWHh2dr3r/++quaNm1qe37p0iWdPHlSrVq1kvS/E1x9fX2zPc/bycv1yY7sfvlJeVNrhQoVbHva/urmeUp/7tG6+Qq15ORknTx5MtN53248s6onq+Xv379fPj4+KlasWJb1SMr00OPtXudff/3V7n/oBw8eVFpamu3E7ODgYBljVLFiRdv/QnNLhQoVlJaWpl9//VXVqlWztSckJOjChQu21ySvOTo6qnnz5mrevLkmTpyoMWPGaOjQoVq3bl2W21F67YcOHbLbk3Hz+KVfUZaampqtbfLnn3+WMcZu3LK7TUp/bgM37z2QlGFbN8bo4MGDtw1hixcvVlRUlCZMmGBru3btWoZlBwcH6+eff77lvLLy122/WbNmdn87cOBAhu0gODhYr7/+ul5//XX9+uuvqlOnjiZMmKCPP/7Y1ueRRx7RI488onfeeUf/+c9/1KVLFy1YsEA9evS4ZQ2//vqr3et3+vTpDHto0vcQXbhwwe5CgTs99H87aWlpOnz4sN3775dffpEkuwsonJ2d1aZNG7Vp00ZpaWnq3bu3Zs6cqeHDh+fqfwDzC4fG7kGtWrVSSkqKpk+fbmtLTU3V5MmT7fr5+vrqscce08yZMzP9ovzrZeTpZs2aZXc8e/r06UpJSVHLli0lSREREfL09NSYMWMyPe6d2TwLcn2yo1ixYlle/p4ftbZq1UqbN2/Wli1b7P7+ySefZJguODhY3377rV3brFmzsvwf4O3GMzMBAQGqU6eO5s2bZ/e6/Pzzz1q9erVdiAoODlZiYqJ+/PFHW9vJkyczXGUl3f51njp1qt3z9Nc0vda///3vcnJy0qhRozL8T9UYo7Nnz2Y579tJX6dJkybZtU+cOFGSMr0CK7edO3cuQ1v6z3/c6nBK+uvzr3/9y6795nVxcnLSM888o88//zzToHDzNvnHH39o8eLFtrYrV65o1qxZGaYLDg7W5s2b7a5o+vrrr7O8JP6jjz7SxYsXbc8XL16skydP3nKbTK//5nGfPHlyhm3/mWee0e7duzPdBm+3h6NevXry9fXVjBkz7F7zFStWaN++fbbt4MqVKxlu7RAcHCwPDw/bdOfPn8+wvOyMZ3h4uIoWLarJkyfbTX/zeKYvU5LdZ0L6bSryypQpU2z/NsZoypQpKlq0qJo3by5JGd6Hjo6OtpB7u8OC9yr2CN2D2rRpo4YNG2rQoEE6evSoqlevriVLltgdJ083depUNWrUSDVr1lTPnj1VqVIlJSQkaNOmTfrtt9+0e/duu/7Jyclq3ry57dLJadOmqVGjRmrbtq0kydPTU9OnT1fXrl31t7/9TZ06dVLp0qV1/PhxLVu2TA0bNrR7oxT0+mRHSEiIpk+frtGjR6ty5cry9fXN8L/BvKz1jTfe0Pz58/XEE0+of//+tsvnK1SoYBcwpD8vk37llVf0zDPP6PHHH9fu3bu1atUqu0uC/+p245mV9957Ty1btlRYWJhefPFF2+XzXl5edvcx6tSpk9588009/fTTevXVV223UnjggQfsTmBNf53XrFmjiRMnqkyZMqpYsaLtpExJOnLkiNq2basnnnhCmzZtsl0OXrt2bUl/fuiPHj1agwcP1tGjR9WuXTt5eHjoyJEjWrp0qV566SUNHDjwluuVldq1aysqKkqzZs3ShQsX1KRJE23ZskXz5s1Tu3bt7Paq5ZW33npL3377rVq3bq0KFSro1KlTmjZtmsqVK6dGjRplOV2dOnXUuXNnTZs2TYmJiWrQoIHi4uJ08ODBDH3Hjh2rdevWKTQ0VD179lT16tV17tw57dixQ2vWrLGFsZ49e2rKlCmKjIzU9u3bFRAQoPnz59suUf+rHj16aPHixXriiSfUoUMHHTp0SB9//HGGc5bSlSxZUo0aNVL37t2VkJCgSZMmqXLlyurZs+ctX58nn3xS8+fPl5eXl6pXr65NmzZpzZo1KlWqlF2/f/zjH1q8eLGeffZZvfDCCwoJCdG5c+f05ZdfasaMGbbtKTNFixbVu+++q+7du6tJkybq3Lmz7fL5oKAgDRgwQNKfe0HS31fVq1dXkSJFtHTpUiUkJKhTp06SpHnz5mnatGl6+umnFRwcrIsXL2r27Nny9PS85R7Z9HutxcbG6sknn1SrVq20c+dOrVixIsP7vEWLFipfvrxefPFF/eMf/5CTk5M+/PBD22dybnN1ddXKlSsVFRWl0NBQrVixQsuWLdOQIUNs54L26NFD586dU7NmzVSuXDkdO3ZMkydPVp06dez2thYq+XuRmnVkdfn8zZf1GpP5JZJnz541Xbt2NZ6ensbLy8t07drV7Ny5M8Ml3MYYc+jQIRMZGWn8/f1N0aJFTdmyZc2TTz5pFi9enKGe//u//zMvvfSSKVGihClevLjp0qWL3SXU6datW2ciIiKMl5eXcXV1NcHBwaZbt25m27Zt99T63Hzpavpls3+9LDQ+Pt60bt3aeHh4GEm3vZQ+t2s1xpgff/zRNGnSxLi6upqyZcuat99+23zwwQcZLoFNTU01b775pvHx8THu7u4mIiLCHDx4MMvL57M7nplZs2aNadiwoXFzczOenp6mTZs2Zu/evRn6rV692tSoUcM4OzubqlWrmo8//jjTMd6/f7959NFHjZubm91lwOl99+7da9q3b288PDxMiRIlTN++fe0uYU73+eefm0aNGplixYqZYsWKmQcffND06dPHHDhwwNanSZMm5qGHHsp0vbLaLm/cuGFGjRplKlasaIoWLWoCAwPN4MGD7W5/YMyflyG3bt06w/Tp29Znn31m157Vtpi+3umXHcfFxZmnnnrKlClTxjg7O5syZcqYzp07m19++SXT9firq1evmldffdWUKlXKFCtWzLRp08acOHEi09stJCQkmD59+pjAwEBTtGhR4+/vb5o3b25mzZpl1+/YsWOmbdu2xt3d3fj4+Jj+/fublStXZnj/GGPMhAkTTNmyZY2Li4tp2LCh2bZtW5aXz3/66adm8ODBxtfX17i5uZnWrVvbXbqdlfPnz5vu3bsbHx8fU7x4cRMREWH279+f6eX7Z8+eNX379jVly5Y1zs7Oply5ciYqKsp224CsxirdwoULTd26dY2Li4spWbKk6dKli/ntt99sfz9z5ozp06ePefDBB02xYsWMl5eXCQ0NtbvdwI4dO0znzp1N+fLljYuLi/H19TVPPvmk3WdkVlJTU82oUaNMQECAcXNzM4899pj5+eefM13X7du3m9DQUOPs7GzKly9vJk6cmOXl85ltt5JMnz597NqOHDmS4fYA6e+bQ4cOmRYtWhh3d3fj5+dnYmJiTGpqqq3f4sWLTYsWLYyvr6+tppdfftmcPHnytut9r3IwJo/PKsU9If0mYlu3blW9evUKuhzcpcI0niNHjtSoUaN0+vTpLPdsofBbv369mjZtqs8++8zusnwUDt26ddPixYt16dKlgi4l33GOEAAAsCyCEAAAsCyCEAAAsKwCPUfo22+/1Xvvvaft27fbLslt167dLadZv369oqOjtWfPHgUGBmrYsGGZ/jIyAADA7RToHqHLly+rdu3aGe4vkpUjR46odevWatq0qXbt2qXXXntNPXr00KpVq/K4UgAAcD+6Z64ac3BwuO0eoTfffFPLli2zu1lYp06ddOHChUL/WycAACD/FaobKm7atCnDreMjIiJu+au9169ft7vbZVpams6dO6dSpUrl6KcXAABAwTHG6OLFiypTpowcHXPvgFahCkLx8fHy8/Oza/Pz81NSUpKuXr2a6Q/sxcbGatSoUflVIgAAyEMnTpxQuXLlcm1+hSoI3YnBgwcrOjra9jwxMVHly5fXiRMn5OnpWYCVAQCA7EpKSlJgYKA8PDxydb6FKgj5+/srISHBri0hIUGenp6Z7g2SJBcXF7m4uGRo9/T0JAgBAFDI5PZpLYXqPkJhYWGKi4uza/vmm28UFhZWQBUBAIDCrECD0KVLl7Rr1y7t2rVL0p+Xx+/atcv2q7qDBw9WZGSkrf8rr7yiw4cP64033tD+/fs1bdo0LVq0yPaLwQAAADlRoEFo27Ztqlu3rurWrStJio6OVt26dTVixAhJ0smTJ22hSJIqVqyoZcuW6ZtvvlHt2rU1YcIE/fvf/1ZERESB1A8AAAq3e+Y+QvklKSlJXl5eSkxM5BwhAADyUWpqqm7cuJHl352dnbO8ND6vvr8L1cnSAACg8DHGKD4+XhcuXLhlP0dHR1WsWFHOzs75U5gIQgAAII+lhyBfX1+5u7tneuVXWlqa/vjjD508eVLly5fPt5seE4QAAECeSU1NtYWgUqVK3bJv6dKl9ccffyglJUVFixbNl/oK1eXzAACgcEk/J8jd3f22fdMPiaWmpuZpTX9FEAIAAHkuO4e6CuI3QAlCAADAsghCAADAsghCAADAsghCAAAgz2Xn/s0FcY9nghAAAMgz6ZfBX7ly5bZ9k5OTJUlOTk55WtNfcR8hAACQZ5ycnOTt7a1Tp05J0i1vqHj69Gm5u7urSJH8iycEIQAAkKf8/f0lyRaGsuLo6Jivd5WWCEIAACCPOTg4KCAgQL6+vnf8o6t5hSAEAADyhZOTU76e/5MdnCwNAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsq8CD0NSpUxUUFCRXV1eFhoZqy5Ytt+w/adIkVa1aVW5ubgoMDNSAAQN07dq1fKoWAADcTwo0CC1cuFDR0dGKiYnRjh07VLt2bUVEROjUqVOZ9v/Pf/6jQYMGKSYmRvv27dMHH3yghQsXasiQIflcOQAAuB8UaBCaOHGievbsqe7du6t69eqaMWOG3N3d9eGHH2ba//vvv1fDhg313HPPKSgoSC1atFDnzp1vuxcJAAAgMwUWhJKTk7V9+3aFh4f/rxhHR4WHh2vTpk2ZTtOgQQNt377dFnwOHz6s5cuXq1WrVlku5/r160pKSrJ7AAAASFKRglrwmTNnlJqaKj8/P7t2Pz8/7d+/P9NpnnvuOZ05c0aNGjWSMUYpKSl65ZVXbnloLDY2VqNGjcrV2gEAwP2hwE+Wzon169drzJgxmjZtmnbs2KElS5Zo2bJlevvtt7OcZvDgwUpMTLQ9Tpw4kY8VAwCAe1mB7RHy8fGRk5OTEhIS7NoTEhLk7++f6TTDhw9X165d1aNHD0lSzZo1dfnyZb300ksaOnSoHB0z5joXFxe5uLjk/goAAIBCr8D2CDk7OyskJERxcXG2trS0NMXFxSksLCzTaa5cuZIh7Dg5OUmSjDF5VywAALgvFdgeIUmKjo5WVFSU6tWrp/r162vSpEm6fPmyunfvLkmKjIxU2bJlFRsbK0lq06aNJk6cqLp16yo0NFQHDx7U8OHD1aZNG1sgAgAAyK4CDUIdO3bU6dOnNWLECMXHx6tOnTpauXKl7QTq48eP2+0BGjZsmBwcHDRs2DD9/vvvKl26tNq0aaN33nmnoFYBAAAUYg7GYseUkpKS5OXlpcTERHl6ehZ0OQAAIBvy6vu7UF01BgAAkJsIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIIQgAAwLIKPAhNnTpVQUFBcnV1VWhoqLZs2XLL/hcuXFCfPn0UEBAgFxcXPfDAA1q+fHk+VQsAAO4nRQpy4QsXLlR0dLRmzJih0NBQTZo0SRERETpw4IB8fX0z9E9OTtbjjz8uX19fLV68WGXLltWxY8fk7e2d/8UDAIBCz8EYYwpq4aGhoXr44Yc1ZcoUSVJaWpoCAwPVr18/DRo0KEP/GTNm6L333tP+/ftVtGjRO1pmUlKSvLy8lJiYKE9Pz7uqHwAA5I+8+v4usENjycnJ2r59u8LDw/9XjKOjwsPDtWnTpkyn+fLLLxUWFqY+ffrIz89PNWrU0JgxY5Samprlcq5fv66kpCS7BwAAgFSAQejMmTNKTU2Vn5+fXbufn5/i4+Mznebw4cNavHixUlNTtXz5cg0fPlwTJkzQ6NGjs1xObGysvLy8bI/AwMBcXQ8AAFB4FfjJ0jmRlpYmX19fzZo1SyEhIerYsaOGDh2qGTNmZDnN4MGDlZiYaHucOHEiHysGAAD3sgI7WdrHx0dOTk5KSEiwa09ISJC/v3+m0wQEBKho0aJycnKytVWrVk3x8fFKTk6Ws7NzhmlcXFzk4uKSu8UDAID7wh3tEfruu+/0/PPPKywsTL///rskaf78+dqwYUO25+Hs7KyQkBDFxcXZ2tLS0hQXF6ewsLBMp2nYsKEOHjyotLQ0W9svv/yigICATEMQAADAreQ4CH3++eeKiIiQm5ubdu7cqevXr0uSEhMTNWbMmBzNKzo6WrNnz9a8efO0b98+9erVS5cvX1b37t0lSZGRkRo8eLCtf69evXTu3Dn1799fv/zyi5YtW6YxY8aoT58+OV0NAACAnB8aGz16tGbMmKHIyEgtWLDA1t6wYcNbnrScmY4dO+r06dMaMWKE4uPjVadOHa1cudJ2AvXx48fl6Pi/rBYYGKhVq1ZpwIABqlWrlsqWLav+/fvrzTffzOlqAAAA5Pw+Qu7u7tq7d6+CgoLk4eGh3bt3q1KlSjp8+LCqV6+ua9eu5VWtuYL7CAEAUPjcM/cR8vf318GDBzO0b9iwQZUqVcqVogAAAPJDjoNQz5491b9/f/3www9ycHDQH3/8oU8++UQDBw5Ur1698qJGAACAPJHjc4QGDRqktLQ0NW/eXFeuXNGjjz4qFxcXDRw4UP369cuLGgEAAPJEjs4RSk1N1caNG1WrVi25u7vr4MGDunTpkqpXr67ixYvnZZ25hnOEAAAofPLq+ztHe4ScnJzUokUL7du3T97e3qpevXquFQIAAJDfcnyOUI0aNXT48OG8qAUAACBf5TgIjR49WgMHDtTXX3+tkydP8svuAACg0MrxfYT+eoNDBwcH27+NMXJwcFBqamruVZcHOEcIAIDC5544R0iS1q1bl2sLBwAAKEg5DkJNmjTJizoAAADyXY6DkCRduHBBH3zwgfbt2ydJeuihh/TCCy/Iy8srV4sDAADISzk+WXrbtm0KDg7WP//5T507d07nzp3TxIkTFRwcrB07duRFjQAAAHkixydLN27cWJUrV9bs2bNVpMifO5RSUlLUo0cPHT58WN9++22eFJpbOFkaAIDCJ6++v3MchNzc3LRz5049+OCDdu179+5VvXr1dOXKlVwrLi8QhAAAKHzumV+f9/T01PHjxzO0nzhxQh4eHrlSFAAAQH7IcRDq2LGjXnzxRS1cuFAnTpzQiRMntGDBAvXo0UOdO3fOixoBAADyRI6vGhs/frwcHBwUGRmplJQUSVLRokXVq1cvjR07NtcLBAAAyCs5Pkco3ZUrV3To0CFJUnBwsNzd3XO1sLzCOUIAABQ+98ydpRMTE5WamqqSJUuqZs2atvZz586pSJEihAsAAFBo5PgcoU6dOmnBggUZ2hctWqROnTrlSlEAAAD5IcdB6IcfflDTpk0ztD/22GP64YcfcqUoAACA/JDjIHT9+nXbSdJ/dePGDV29ejVXigIAAMgPOQ5C9evX16xZszK0z5gxQyEhIblSFAAAQH7I8cnSo0ePVnh4uHbv3q3mzZtLkuLi4rR161atXr061wsEAADIKzneI9SwYUNt2rRJgYGBWrRokb766itVrlxZP/74oxo3bpwXNQIAAOSJO76PUGHFfYQAACh8Cvw+QikpKUpNTZWLi4utLSEhQTNmzNDly5fVtm1bNWrUKNcKAwAAyGvZDkI9e/aUs7OzZs6cKUm6ePGiHn74YV27dk0BAQH65z//qf/+979q1apVnhULAACQm7J9jtDGjRv1zDPP2J5/9NFHSk1N1a+//qrdu3crOjpa7733Xp4UCQAAkBeyHYR+//13ValSxfY8Li5OzzzzjLy8vCRJUVFR2rNnT+5XCAAAkEeyHYRcXV3tbpi4efNmhYaG2v390qVLuVsdAABAHsp2EKpTp47mz58vSfruu++UkJCgZs2a2f5+6NAhlSlTJvcrBAAAyCPZPll6xIgRatmypRYtWqSTJ0+qW7duCggIsP196dKlatiwYZ4UCQAAkBeyHYSaNGmi7du3a/Xq1fL399ezzz5r9/c6deqofv36uV4gAABAXuGGigAA4J6XV9/fOf6JDQAAgPsFQQgAAFgWQQgAAFgWQQgAAFgWQQgAAFhWti+fT1eiRAk5ODhkaHdwcJCrq6sqV66sbt26qXv37rlSIAAAQF7JcRAaMWKE3nnnHbVs2dJ236AtW7Zo5cqV6tOnj44cOaJevXopJSVFPXv2zPWCAQAAckuOg9CGDRs0evRovfLKK3btM2fO1OrVq/X555+rVq1a+te//kUQAgAA97QcnyO0atUqhYeHZ2hv3ry5Vq1aJUlq1aqVDh8+fPfVAQAA5KEcB6GSJUvqq6++ytD+1VdfqWTJkpKky5cvy8PD4+6rAwAAyEM5PjQ2fPhw9erVS+vWrbOdI7R161YtX75cM2bMkCR98803atKkSe5WCgAAkMvu6LfGNm7cqClTpujAgQOSpKpVq6pfv35q0KBBrheY2/itMQAACp+8+v7mR1cBAMA9L6++v3N8aCwpKSnTdgcHB7m4uMjZ2fmuiwIAAMgPOQ5C3t7emd5QMV25cuXUrVs3xcTEyNGRG1cDAIB7V46D0Ny5czV06FB169bN7oaK8+bN07Bhw3T69GmNHz9eLi4uGjJkSK4XDAAAkFtyHITmzZunCRMmqEOHDra2Nm3aqGbNmpo5c6bi4uJUvnx5vfPOOwQhAABwT8vxsavvv/9edevWzdBet25dbdq0SZLUqFEjHT9+/O6rAwAAyEM5DkKBgYH64IMPMrR/8MEHCgwMlCSdPXtWJUqUuPvqAAAA8lCOD42NHz9ezz77rFasWKGHH35YkrRt2zbt379fixcvlvTnDRY7duyYu5UCAADksju6j9DRo0c1c+ZMuxsqvvzyywoKCsrt+nId9xECAKDwKRQ3VPz5559Vo0aN3JpdniAIAQBQ+OTV9/dd3+jn4sWLmjVrlurXr6/atWvnRk0AAAD54o6D0LfffquoqCgFBARo/PjxatasmTZv3pybtQEAAOSpHJ0sHR8fr7lz5+qDDz5QUlKSOnTooOvXr+uLL75Q9erV86pGAACAPJHtPUJt2rRR1apV9eOPP2rSpEn6448/NHny5LysDQAAIE9le4/QihUr9Oqrr6pXr16qUqVKXtYEAACQL7K9R2jDhg26ePGiQkJCFBoaqilTpujMmTN5WRsAAECeynYQeuSRRzR79mydPHlSL7/8shYsWKAyZcooLS1N33zzjS5evHjHRUydOlVBQUFydXVVaGiotmzZkq3pFixYIAcHB7Vr1+6Olw0AAKwrx1eNFStWTC+88II2bNign376Sa+//rrGjh0rX19ftW3bNscFLFy4UNHR0YqJidGOHTtUu3ZtRURE6NSpU7ec7ujRoxo4cKAaN26c42UCAABId3kfoapVq2rcuHH67bff9Omnn97RPCZOnKiePXuqe/fuql69umbMmCF3d3d9+OGHWU6TmpqqLl26aNSoUapUqdKdlg8AACzurm+oKElOTk5q166dvvzyyxxNl5ycrO3btys8PPx/BTk6Kjw83PZL9pl566235OvrqxdffPG2y7h+/bqSkpLsHgAAAFIuBaE7debMGaWmpsrPz8+u3c/PT/Hx8ZlOs2HDBn3wwQeaPXt2tpYRGxsrLy8v2yMwMPCu6wYAAPeHAg1COXXx4kV17dpVs2fPlo+PT7amGTx4sBITE22PEydO5HGVAACgsMjRnaVzm4+Pj5ycnJSQkGDXnpCQIH9//wz9Dx06pKNHj6pNmza2trS0NElSkSJFdODAAQUHB9tN4+LiIhcXlzyoHgAAFHYFukfI2dlZISEhiouLs7WlpaUpLi5OYWFhGfo/+OCD+umnn7Rr1y7bo23btmratKl27drFYS8AAJAjBbpHSJKio6MVFRWlevXqqX79+po0aZIuX76s7t27S5IiIyNVtmxZxcbGytXVVTVq1LCb3tvbW5IytAMAANxOgQehjh076vTp0xoxYoTi4+NVp04drVy50nYC9fHjx+XoWKhOZQIAAIWEgzHGFHQR+SkpKUleXl5KTEyUp6dnQZcDAACyIa++v9nVAgAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALIsgBAAALOueCEJTp05VUFCQXF1dFRoaqi1btmTZd/bs2WrcuLFKlCihEiVKKDw8/Jb9AQAAslLgQWjhwoWKjo5WTEyMduzYodq1aysiIkKnTp3KtP/69evVuXNnrVu3Tps2bVJgYKBatGih33//PZ8rBwAAhZ2DMcYUZAGhoaF6+OGHNWXKFElSWlqaAgMD1a9fPw0aNOi206empqpEiRKaMmWKIiMjb9s/KSlJXl5eSkxMlKen513XDwAA8l5efX8X6B6h5ORkbd++XeHh4bY2R0dHhYeHa9OmTdmax5UrV3Tjxg2VLFky079fv35dSUlJdg8AAACpgIPQmTNnlJqaKj8/P7t2Pz8/xcfHZ2seb775psqUKWMXpv4qNjZWXl5etkdgYOBd1w0AAO4PBX6O0N0YO3asFixYoKVLl8rV1TXTPoMHD1ZiYqLtceLEiXyuEgAA3KuKFOTCfXx85OTkpISEBLv2hIQE+fv733La8ePHa+zYsVqzZo1q1aqVZT8XFxe5uLjkSr0AAOD+UqB7hJydnRUSEqK4uDhbW1pamuLi4hQWFpbldOPGjdPbb7+tlStXql69evlRKgAAuA8V6B4hSYqOjlZUVJTq1aun+vXra9KkSbp8+bK6d+8uSYqMjFTZsmUVGxsrSXr33Xc1YsQI/ec//1FQUJDtXKLixYurePHiBbYeAACg8CnwINSxY0edPn1aI0aMUHx8vOrUqaOVK1faTqA+fvy4HB3/t+Nq+vTpSk5OVvv27e3mExMTo5EjR+Zn6QAAoJAr8PsI5TfuIwQAQOFzX95HCAAAoCARhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGXdE0Fo6tSpCgoKkqurq0JDQ7Vly5Zb9v/ss8/04IMPytXVVTVr1tTy5cvzqVIAAHA/KfAgtHDhQkVHRysmJkY7duxQ7dq1FRERoVOnTmXa//vvv1fnzp314osvaufOnWrXrp3atWunn3/+OZ8rBwAAhZ2DMcYUZAGhoaF6+OGHNWXKFElSWlqaAgMD1a9fPw0aNChD/44dO+ry5cv6+uuvbW2PPPKI6tSpoxkzZtx2eUlJSfLy8lJiYqI8PT1zb0UAAECeyavv7wLdI5ScnKzt27crPDzc1ubo6Kjw8HBt2rQp02k2bdpk11+SIiIisuwPAACQlSIFufAzZ84oNTVVfn5+du1+fn7av39/ptPEx8dn2j8+Pj7T/tevX9f169dtzxMTEyX9mSwBAEDhkP69ndsHsgo0COWH2NhYjRo1KkN7YGBgAVQDAADuxtmzZ+Xl5ZVr8yvQIOTj4yMnJyclJCTYtSckJMjf3z/Tafz9/XPUf/DgwYqOjrY9v3DhgipUqKDjx4/n6guJnEtKSlJgYKBOnDjB+Vr3AMbj3sFY3DsYi3tHYmKiypcvr5IlS+bqfAs0CDk7OyskJERxcXFq166dpD9Plo6Li1Pfvn0znSYsLExxcXF67bXXbG3ffPONwsLCMu3v4uIiFxeXDO1eXl5s1PcIT09PxuIewnjcOxiLewdjce9wdMzd05sL/NBYdHS0oqKiVK9ePdWvX1+TJk3S5cuX1b17d0lSZGSkypYtq9jYWElS//791aRJE02YMEGtW7fWggULtG3bNs2aNasgVwMAABRCBR6EOnbsqNOnT2vEiBGKj49XnTp1tHLlStsJ0cePH7dLfw0aNNB//vMfDRs2TEOGDFGVKlX0xRdfqEaNGgW1CgAAoJAq8CAkSX379s3yUNj69esztD377LN69tln72hZLi4uiomJyfRwGfIXY3FvYTzuHYzFvYOxuHfk1VgU+A0VAQAACkqB/8QGAABAQSEIAQAAyyIIAQAAyyIIAQAAy7ovg9DUqVMVFBQkV1dXhYaGasuWLbfs/9lnn+nBBx+Uq6uratasqeXLl+dTpfe/nIzF7Nmz1bhxY5UoUUIlSpRQeHj4bccOOZPT90a6BQsWyMHBwXbjU9y9nI7FhQsX1KdPHwUEBMjFxUUPPPAAn1W5JKdjMWnSJFWtWlVubm4KDAzUgAEDdO3atXyq9v717bffqk2bNipTpowcHBz0xRdf3Haa9evX629/+5tcXFxUuXJlzZ07N+cLNveZBQsWGGdnZ/Phhx+aPXv2mJ49expvb2+TkJCQaf+NGzcaJycnM27cOLN3714zbNgwU7RoUfPTTz/lc+X3n5yOxXPPPWemTp1qdu7cafbt22e6detmvLy8zG+//ZbPld+fcjoe6Y4cOWLKli1rGjdubJ566qn8KfY+l9OxuH79uqlXr55p1aqV2bBhgzly5IhZv3692bVrVz5Xfv/J6Vh88sknxsXFxXzyySfmyJEjZtWqVSYgIMAMGDAgnyu//yxfvtwMHTrULFmyxEgyS5cuvWX/w4cPG3d3dxMdHW327t1rJk+ebJycnMzKlStztNz7LgjVr1/f9OnTx/Y8NTXVlClTxsTGxmbav0OHDqZ169Z2baGhoebll1/O0zqtIKdjcbOUlBTj4eFh5s2bl1clWsqdjEdKSopp0KCB+fe//22ioqIIQrkkp2Mxffp0U6lSJZOcnJxfJVpGTseiT58+plmzZnZt0dHRpmHDhnlap9VkJwi98cYb5qGHHrJr69ixo4mIiMjRsu6rQ2PJycnavn27wsPDbW2Ojo4KDw/Xpk2bMp1m06ZNdv0lKSIiIsv+yJ47GYubXblyRTdu3Mj1H9izojsdj7feeku+vr568cUX86NMS7iTsfjyyy8VFhamPn36yM/PTzVq1NCYMWOUmpqaX2Xfl+5kLBo0aKDt27fbDp8dPnxYy5cvV6tWrfKlZvxPbn1/3xN3ls4tZ86cUWpqqu3nOdL5+flp//79mU4THx+faf/4+Pg8q9MK7mQsbvbmm2+qTJkyGTZ05NydjMeGDRv0wQcfaNeuXflQoXXcyVgcPnxYa9euVZcuXbR8+XIdPHhQvXv31o0bNxQTE5MfZd+X7mQsnnvuOZ05c0aNGjWSMUYpKSl65ZVXNGTIkPwoGX+R1fd3UlKSrl69Kjc3t2zN577aI4T7x9ixY7VgwQItXbpUrq6uBV2O5Vy8eFFdu3bV7Nmz5ePjU9DlWF5aWpp8fX01a9YshYSEqGPHjho6dKhmzJhR0KVZzvr16zVmzBhNmzZNO3bs0JIlS7Rs2TK9/fbbBV0a7tB9tUfIx8dHTk5OSkhIsGtPSEiQv79/ptP4+/vnqD+y507GIt348eM1duxYrVmzRrVq1crLMi0jp+Nx6NAhHT16VG3atLG1paWlSZKKFCmiAwcOKDg4OG+Lvk/dyXsjICBARYsWlZOTk62tWrVqio+PV3JyspydnfO05vvVnYzF8OHD1bVrV/Xo0UOSVLNmTV2+fFkvvfSShg4davcj4chbWX1/e3p6ZntvkHSf7RFydnZWSEiI4uLibG1paWmKi4tTWFhYptOEhYXZ9Zekb775Jsv+yJ47GQtJGjdunN5++22tXLlS9erVy49SLSGn4/Hggw/qp59+0q5du2yPtm3bqmnTptq1a5cCAwPzs/z7yp28Nxo2bKiDBw/awqgk/fLLLwoICCAE3YU7GYsrV65kCDvpAdXw0535Kte+v3N2Hve9b8GCBcbFxcXMnTvX7N2717z00kvG29vbxMfHG2OM6dq1qxk0aJCt/8aNG02RIkXM+PHjzb59+0xMTAyXz+eSnI7F2LFjjbOzs1m8eLE5efKk7XHx4sWCWoX7Sk7H42ZcNZZ7cjoWx48fNx4eHqZv377mwIED5uuvvza+vr5m9OjRBbUK942cjkVMTIzx8PAwn376qTl8+LBZvXq1CQ4ONh06dCioVbhvXLx40ezcudPs3LnTSDITJ040O3fuNMeOHTPGGDNo0CDTtWtXW//0y+f/8Y9/mH379pmpU6dy+Xy6yZMnm/LlyxtnZ2dTv359s3nzZtvfmjRpYqKiouz6L1q0yDzwwAPG2dnZPPTQQ2bZsmX5XPH9KydjUaFCBSMpwyMmJib/C79P5fS98VcEodyV07H4/vvvTWhoqHFxcTGVKlUy77zzjklJScnnqu9PORmLGzdumJEjR5rg4GDj6upqAgMDTe/evc358+fzv/D7zLp16zL9Dkh//aOiokyTJk0yTFOnTh3j7OxsKlWqZObMmZPj5ToYw748AABgTffVOUIAAAA5QRACAACWRRACAACWRRACAACWRRACAACWRRACAACWRRACAACWRRAC7lNBQUGaNGlStvuvX79eDg4OunDhQp7VdDfmzp0rb2/vgi4jg7i4OFWrVk2pqamSpJEjR6pOnTq3nKZbt25q166d7fljjz2m1157zfb85rFzcHDQF198kXtF30anTp00YcKEfFseUJAIQkABc3BwuOVj5MiRdzTfrVu36qWXXsp2/wYNGujkyZPy8vK6o+Vl170aaO7UG2+8oWHDhtn9IOrtvP/++5o7d262+588eVItW7a8g+ruzLBhw/TOO+8oMTEx35YJFJT76tfngcLo5MmTtn8vXLhQI0aM0IEDB2xtxYsXt/3bGKPU1FQVKXL7t27p0qVzVIezs3OWv7iNzG3YsEGHDh3SM888k6Ppcho283tcatSooeDgYH388cfq06dPvi4byG/sEQIKmL+/v+3h5eUlBwcH2/P9+/fLw8NDK1asUEhIiFxcXGxfvk899ZT8/PxUvHhxPfzww1qzZo3dfDM7vPLvf/9bTz/9tNzd3VWlShV9+eWXtr/ffGgsfc/NqlWrVK1aNRUvXlxPPPGEXXBLSUnRq6++Km9vb5UqVUpvvvmmoqKi7A77/NX69evVvXt3JSYmZtjjdf78eUVGRqpEiRJyd3dXy5Yt9euvv2b5up0+fVr16tXT008/revXrystLU2xsbGqWLGi3NzcVLt2bS1evDjD+sXFxalevXpyd3dXgwYN7ELn7t271bRpU3l4eMjT01MhISHatm1bljUsWLBAjz/+uFxdXTP8bebMmQoMDJS7u7s6dOhgt3fl5kNjt3PzobGffvpJzZo1k5ubm0qVKqWXXnpJly5dyjD/8ePHKyAgQKVKlVKfPn1048YNW59p06apSpUqcnV1lZ+fn9q3b2+3zDZt2mjBggXZrhEorAhCQCEwaNAgjR07Vvv27VOtWrV06dIltWrVSnFxcdq5c6eeeOIJtWnTRsePH7/lfEaNGqUOHTroxx9/VKtWrdSlSxedO3cuy/5XrlzR+PHjNX/+fH377bc6fvy4Bg4caPv7u+++q08++URz5szRxo0blZSUdMtzWRo0aKBJkybJ09NTJ0+e1MmTJ23z69atm7Zt26Yvv/xSmzZtkjFGrVq1svvyTnfixAk1btxYNWrU0OLFi+Xi4qLY2Fh99NFHmjFjhvbs2aMBAwbo+eef1//93//ZTTt06FBNmDBB27ZtU5EiRfTCCy/Y/talSxeVK1dOW7du1fbt2zVo0CAVLVo0y/X57rvvVK9evQztBw8e1KJFi/TVV19p5cqV2rlzp3r37p3lfHLi8uXLioiIUIkSJbR161Z99tlnWrNmjfr27WvXb926dTp06JDWrVunefPmae7cubbDcdu2bdOrr76qt956SwcOHNDKlSv16KOP2k1fv359bdmyRdevX8+VuoF71t39ViyA3DRnzhzj5eVle57+a8xffPHFbad96KGHzOTJk23PK1SoYP75z3/anksyw4YNsz2/dOmSkWRWrFhht6z0X9GeM2eOkWQOHjxom2bq1KnGz8/P9tzPz8+89957tucpKSmmfPnyt/yV+pvX0RhjfvnlFyPJbNy40dZ25swZ4+bmZhYtWmQ33f79+01gYKB59dVXTVpamjHGmGvXrhl3d3fz/fff2833xRdfNJ07d7ZbvzVr1tj+vmzZMiPJXL161RhjjIeHh5k7d26Wtd/My8vLfPTRR3ZtMTExxsnJyfz222+2thUrVhhHR0dz8uRJY8yfv6L919eoSZMmpn///rbnmY3d0qVLjTHGzJo1y5QoUcJcunTJbj0cHR1NfHy8bf4VKlSw+3X6Z5991nTs2NEYY8znn39uPD09TVJSUpbrtnv3biPJHD16NHsvBlBIsUcIKARu3utw6dIlDRw4UNWqVZO3t7eKFy+uffv23XaPUK1atWz/LlasmDw9PXXq1Kks+7u7uys4ONj2PCAgwNY/MTFRCQkJql+/vu3vTk5OCgkJydG6SdK+fftUpEgRhYaG2tpKlSqlqlWrat++fba2q1evqnHjxvr73/+u999/Xw4ODpL+3ANz5coVPf744ypevLjt8dFHH+nQoUNZvgYBAQGSZFun6Oho9ejRQ+Hh4Ro7dmyGaW929erVTA+LlS9fXmXLlrU9DwsLU1pamt1huDu1b98+1a5dW8WKFbO1NWzYMMP8H3roIbsTuP86do8//rgqVKigSpUqqWvXrvrkk0905coVu+W4ublJUoZ24H5DEAIKgb9+6UnSwIEDtXTpUo0ZM0bfffeddu3apZo1ayo5OfmW87n5MI+Dg4PS0tJy1N8Yk8Pqc4+Li4vCw8P19ddf6/fff7e1p58fs2zZMu3atcv22Lt3r915QpL9OqUHqfTXYOTIkdqzZ49at26ttWvXqnr16lq6dGmW9fj4+Oj8+fO5tn656VZj7eHhoR07dujTTz9VQECARowYodq1a9vdOiH9kGlOT7oHChuCEFAIbdy4Ud26ddPTTz+tmjVryt/fX0ePHs3XGry8vOTn56etW7fa2lJTU7Vjx45bTufs7Gy75066atWqKSUlRT/88IOt7ezZszpw4ICqV69ua3N0dNT8+fMVEhKipk2b6o8//pAkVa9eXS4uLjp+/LgqV65s9wgMDMzRej3wwAMaMGCAVq9erb///e+aM2dOln3r1q2rvXv3Zmg/fvy4rTZJ2rx5sxwdHVW1atUc1ZKZatWqaffu3bp8+bKtbePGjTmef5EiRRQeHq5x48bpxx9/1NGjR7V27Vrb33/++WeVK1dOPj4+d10zcC8jCAGFUJUqVbRkyRLt2rVLu3fv1nPPPXfLPTt5pV+/foqNjdV///tfHThwQP3799f58+dte1oyExQUpEuXLikuLk5nzpzRlStXVKVKFT311FPq2bOnNmzYoN27d+v5559X2bJl9dRTT9lN7+TkpE8++US1a9dWs2bNFB8fLw8PDw0cOFADBgzQvHnzdOjQIe3YsUOTJ0/WvHnzsrUuV69eVd++fbV+/XodO3ZMGzdu1NatW1WtWrUsp4mIiNCGDRsytLu6uioqKkq7d+/Wd999p1dffVUdOnTIlcvgu3TpYpv/zz//rHXr1qlfv37q2rWr/Pz8sjWPr7/+Wv/617+0a9cuHTt2TB999JHS0tLsgtR3332nFi1a3HW9wL2OIAQUQhMnTlSJEiXUoEEDtWnTRhEREfrb3/6W73W8+eab6ty5syIjIxUWFqbixYsrIiIi0/Nm0jVo0ECvvPKKOnbsqNKlS2vcuHGSpDlz5igkJERPPvmkwsLCZIzR8uXLM71qq0iRIvr000/10EMPqVmzZjp16pTefvttDR8+XLGxsapWrZqeeOIJLVu2TBUrVszWujg5Oens2bOKjIzUAw88oA4dOqhly5YaNWpUltN06dJFe/bsyXDuT+XKlfX3v/9drVq1UosWLVSrVi1NmzYtW3Xcjru7u1atWqVz587p4YcfVvv27dW8eXNNmTIl2/Pw9vbWkiVL1KxZM1WrVk0zZsywvZ6SdO3aNX3xxRfq2bNnrtQM3MscTEEe8AdwX0lLS1O1atXUoUMHvf322wVdTr74xz/+oaSkJM2cObOgS8k106dP19KlS7V69eqCLgXIc+wRAnDHjh07ptmzZ+uXX37RTz/9pF69eunIkSN67rnnCrq0fDN06FBVqFChQA5N5pWiRYtq8uTJBV0GkC/YIwTgjp04cUKdOnXSzz//LGOMatSoobFjx2a4OR8A3KsIQgAAwLI4NAYAACyLIAQAACyLIAQAACyLIAQAACyLIAQAACyLIAQAACyLIAQAACyLIAQAACyLIAQAACzr/wEEf7gHRKN9xgAAAABJRU5ErkJggg==",
+      "text/plain": [
+       "<Figure size 640x480 with 1 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from matplotlib import pyplot as plt\n",
+    "\n",
+    "import json\n",
+    "import os\n",
+    "from matplotlib import pyplot as plt\n",
+    "metrics = ['agg_score', 'commonsense_qa/acc_norm', 'hellaswag/acc_norm', 'openbookqa/acc_norm', 'piqa/acc_norm',\n",
+    "                   'siqa/acc_norm', 'winogrande/acc_norm', 'arc/acc_norm', 'mmlu/acc_norm']\n",
+    "\n",
+    "def normalize_runname(runname):\n",
+    "    return runname.replace(\"/\", \"_\")\n",
+    "\n",
+    "grouped = (\n",
+    "    df.groupby([\"runname\", \"steps\"])\n",
+    "    .agg(\n",
+    "        {\n",
+    "            key: \"mean\" for key in metrics\n",
+    "        }\n",
+    "    )\n",
+    "    .reset_index()\n",
+    ")\n",
+    "\n",
+    "file_id=\"../assets/data/plots/ind_dedup_better\"\n",
+    "files = {}\n",
+    "for metric in metrics:\n",
+    "    datas = {}\n",
+    "    for name, group in grouped.groupby(\"runname\"):\n",
+    "        group = group[[\"steps\", metric]].sort_values(by=\"steps\")\n",
+    "        group = group.set_index(\"steps\")\n",
+    "        rolling_avg = group\n",
+    "        # rolling_avg = group.rolling(wjjjjjjjjjjjjjindow=5).mean()\n",
+    "        datas[name] = {\n",
+    "            \"x\": (rolling_avg.index * 2048 * 1024 * 1e-9).tolist(),\n",
+    "            \"y\": rolling_avg[metric].tolist(),\n",
+    "            \"label\": runs_mapping[name],\n",
+    "        }\n",
+    "    # Sort the datata based on the steps\n",
+    "    datas = {k: v for k, v in sorted(datas.items(), key=lambda x: -x[1][\"y\"][-1])}\n",
+    "    # Create a folder\n",
+    "    os.makedirs(f\"{file_id}\", exist_ok=True)\n",
+    "    with open(f\"{file_id}/{normalize_runname(metric)}.json\", \"w\") as f:\n",
+    "        json.dump({\n",
+    "            \"data\": datas,\n",
+    "            \"layout\": {\n",
+    "                \"title\": {\n",
+    "                    \"text\": \"Independent dedup outperforms dedup across dumps\"\n",
+    "                },\n",
+    "            }\n",
+    "        }, f)\n",
+    "    files[metric] = {\"file\": f\"{normalize_runname(metric)}.json\"}\n",
+    "# Create index\n",
+    "with open(f\"{file_id}/index.json\", \"w\") as f:\n",
+    "    json.dump({\n",
+    "        \"files\": files,\n",
+    "        \"settings\": {\n",
+    "            \"defaultMetric\": \"agg_score\",\n",
+    "            \"slider\":{\"min\":0,\"max\":30,\"default\":5}\n",
+    "        }\n",
+    "    }, f)\n",
+    "        \n",
+    "\n",
+    "    \n",
+    "# Add labels and legend\n",
+    "plt.xlabel('Training tokens (billions)')\n",
+    "plt.ylabel('Agg Score')\n",
+    "plt.title('Independent dedup outperforms dedup across dumps')\n",
+    "plt.legend()\n",
+    "\n",
+    "# Show the plot\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "af28ebbd054cdc33",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-04-30T15:08:02.522543Z",
+     "start_time": "2024-04-30T15:08:02.520569Z"
+    },
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/plot_dedup_simul.ipynb

@@ -0,0 +1,1420 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "\n",
+    "\n",
+    "def normalize_run_name(run_name):\n",
+    "    return run_name.replace(\"/\", \"_\")\n",
+    "\n",
+    "def save_for_plot(dir_name, df, run_names, xlabel=\"Dataset\", ylabel=\"Matched as dups probability\", plot_name=\"plot name\", custom_layout={}, ranges={}, x_column=None, default_metric=None):\n",
+    "    import os\n",
+    "    files = {}\n",
+    "    os.makedirs(f\"data/plots/{dir_name}\", exist_ok=True)\n",
+    "    data = {}\n",
+    "    for run_name in run_names:\n",
+    "        data[run_name] = {\n",
+    "            \"x\": df[x_column].tolist() if x_column else [run_name],\n",
+    "            \"y\": df[run_name].tolist(),\n",
+    "            \"label\": run_name,\n",
+    "        }\n",
+    "    file_name = f\"default.json\"\n",
+    "    files[\"default\"] = {\"file\": f\"{file_name}\"}\n",
+    "    with open(f\"data/plots/{dir_name}/{file_name}\", \"w\") as f:\n",
+    "        json.dump({\n",
+    "            \"data\": data,\n",
+    "            \"layout\": {\n",
+    "                \"title\": {\n",
+    "                    \"text\": plot_name,\n",
+    "                },\n",
+    "                \"xaxis\": {\n",
+    "                    \"title\": {\n",
+    "                        \"text\": xlabel,\n",
+    "                    },\n",
+    "                },\n",
+    "                \"yaxis\": {\n",
+    "                    # \"range\": ranges.get(view, None),\n",
+    "                    \"title\": {\n",
+    "                        \"text\": ylabel,\n",
+    "                    },\n",
+    "                },\n",
+    "                **custom_layout,\n",
+    "            }\n",
+    "        }, f)\n",
+    "    with open(f\"data/plots/{dir_name}/index.json\", \"w\") as f:\n",
+    "        json.dump({\n",
+    "            \"files\": files,\n",
+    "            \"settings\": {\n",
+    "                \"defaultMetric\": default_metric,\n",
+    "                \"slider\": None,\n",
+    "                \"autoSetXRange\": False,\n",
+    "                \"type\": \"bar\"\n",
+    "            }\n",
+    "        }, f)\n",
+    "    return files\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>1</th>\n",
+       "      <th>2</th>\n",
+       "      <th>3</th>\n",
+       "      <th>4-8</th>\n",
+       "      <th>8-16</th>\n",
+       "      <th>16-32</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>1B</th>\n",
+       "      <td>0.994974</td>\n",
+       "      <td>0.005008</td>\n",
+       "      <td>0.000018</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>10B</th>\n",
+       "      <td>0.951508</td>\n",
+       "      <td>0.047331</td>\n",
+       "      <td>0.001144</td>\n",
+       "      <td>0.000017</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>100B</th>\n",
+       "      <td>0.608873</td>\n",
+       "      <td>0.302822</td>\n",
+       "      <td>0.074548</td>\n",
+       "      <td>0.013757</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>350B</th>\n",
+       "      <td>0.174147</td>\n",
+       "      <td>0.30712</td>\n",
+       "      <td>0.268018</td>\n",
+       "      <td>0.250649</td>\n",
+       "      <td>0.000065</td>\n",
+       "      <td>0.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1T</th>\n",
+       "      <td>0.006232</td>\n",
+       "      <td>0.03247</td>\n",
+       "      <td>0.083743</td>\n",
+       "      <td>0.817636</td>\n",
+       "      <td>0.05991</td>\n",
+       "      <td>0.000008</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "             1         2         3       4-8      8-16     16-32\n",
+       "1B    0.994974  0.005008  0.000018       0.0       0.0       0.0\n",
+       "10B   0.951508  0.047331  0.001144  0.000017       0.0       0.0\n",
+       "100B  0.608873  0.302822  0.074548  0.013757       0.0       0.0\n",
+       "350B  0.174147   0.30712  0.268018  0.250649  0.000065       0.0\n",
+       "1T    0.006232   0.03247  0.083743  0.817636   0.05991  0.000008"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>index</th>\n",
+       "      <th>1</th>\n",
+       "      <th>2</th>\n",
+       "      <th>3</th>\n",
+       "      <th>4-8</th>\n",
+       "      <th>8-16</th>\n",
+       "      <th>16-32</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>1B</td>\n",
+       "      <td>0.994974</td>\n",
+       "      <td>0.005008</td>\n",
+       "      <td>0.000018</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>10B</td>\n",
+       "      <td>0.951508</td>\n",
+       "      <td>0.047331</td>\n",
+       "      <td>0.001144</td>\n",
+       "      <td>0.000017</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>100B</td>\n",
+       "      <td>0.608873</td>\n",
+       "      <td>0.302822</td>\n",
+       "      <td>0.074548</td>\n",
+       "      <td>0.013757</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>350B</td>\n",
+       "      <td>0.174147</td>\n",
+       "      <td>0.30712</td>\n",
+       "      <td>0.268018</td>\n",
+       "      <td>0.250649</td>\n",
+       "      <td>0.000065</td>\n",
+       "      <td>0.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>1T</td>\n",
+       "      <td>0.006232</td>\n",
+       "      <td>0.03247</td>\n",
+       "      <td>0.083743</td>\n",
+       "      <td>0.817636</td>\n",
+       "      <td>0.05991</td>\n",
+       "      <td>0.000008</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "  index         1         2         3       4-8      8-16     16-32\n",
+       "0    1B  0.994974  0.005008  0.000018       0.0       0.0       0.0\n",
+       "1   10B  0.951508  0.047331  0.001144  0.000017       0.0       0.0\n",
+       "2  100B  0.608873  0.302822  0.074548  0.013757       0.0       0.0\n",
+       "3  350B  0.174147   0.30712  0.268018  0.250649  0.000065       0.0\n",
+       "4    1T  0.006232   0.03247  0.083743  0.817636   0.05991  0.000008"
+      ]
+     },
+     "execution_count": 28,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "summarized_df.reset_index()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 57,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'default': {'file': 'default.json'}}"
+      ]
+     },
+     "execution_count": 57,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "\n",
+    "\n",
+    "df = pd.read_csv(\"./data/duplicates-simulation.csv\", index_col=0)\n",
+    "\n",
+    "\n",
+    "def summarize_ranges(df):\n",
+    "    df_summarized = pd.DataFrame(\n",
+    "        index=[\"1\", \"2\", \"3\", \"4-8\", \"8-16\", \"16-32\"], columns=df.columns\n",
+    "    )\n",
+    "    df_summarized.loc[\"1\"] = df.iloc[0]\n",
+    "    df_summarized.loc[\"2\"] = df.iloc[1]\n",
+    "    df_summarized.loc[\"3\"] = df.iloc[2]\n",
+    "    df_summarized.loc[\"4-8\"] = df.iloc[3:9].sum()\n",
+    "    df_summarized.loc[\"8-16\"] = df.iloc[9:17].sum()\n",
+    "    df_summarized.loc[\"16-32\"] = df.iloc[17:].sum()\n",
+    "    return df_summarized\n",
+    "\n",
+    "\n",
+    "summarized_df = summarize_ranges(df).T\n",
+    "cols = summarized_df.columns\n",
+    "summarized_df.reset_index(inplace=True)\n",
+    "save_for_plot(\n",
+    "    \"duplicates-simul\",\n",
+    "    summarized_df,\n",
+    "    cols,\n",
+    "    x_column=\"index\",\n",
+    "    plot_name=\"Sampling from 1000 identical buckets with 200B tokens each\",\n",
+    "    ylabel=\"Dataset fraction\",\n",
+    "    xlabel=\"Sample size\",\n",
+    "    default_metric=\"default\",\n",
+    "    custom_layout={\n",
+    "        \"barmode\": \"stack\",\n",
+    "        \"legend\": {\n",
+    "            \"title\": {\n",
+    "                \"text\": \"# duplicates\",\n",
+    "                \"font\": {\n",
+    "                    \"size\": 14,\n",
+    "                    \"weight\": \"bold\",\n",
+    "                }\n",
+    "            },\n",
+    "            \"font\": {\n",
+    "                \"size\": 14,\n",
+    "            },\n",
+    "            \"bgcolor\": 'rgba(255, 255, 255, 0.9)',\n",
+    "            # \"borderwidth\": 1,\n",
+    "            \"orientation\": \"v\",\n",
+    "            \"xanchor\": \"left\",\n",
+    "            \"yanchor\": \"bottom\",\n",
+    "            \"x\": 0.01,\n",
+    "            \"y\": 0,\n",
+    "        },\n",
+    "    },\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "KeyError",
+     "evalue": "'index'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
+      "File \u001b[0;32m~/.pyenv/versions/3.12.2/envs/datatrove/lib/python3.12/site-packages/pandas/core/indexes/base.py:3805\u001b[0m, in \u001b[0;36mIndex.get_loc\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   3804\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 3805\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_engine\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_loc\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcasted_key\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   3806\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err:\n",
+      "File \u001b[0;32mindex.pyx:167\u001b[0m, in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
+      "File \u001b[0;32mindex.pyx:196\u001b[0m, in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
+      "File \u001b[0;32mpandas/_libs/hashtable_class_helper.pxi:7081\u001b[0m, in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
+      "File \u001b[0;32mpandas/_libs/hashtable_class_helper.pxi:7089\u001b[0m, in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
+      "\u001b[0;31mKeyError\u001b[0m: 'index'",
+      "\nThe above exception was the direct cause of the following exception:\n",
+      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[17], line 3\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m# Take the sumarized_df and pivot it\u001b[39;00m\n\u001b[0;32m----> 3\u001b[0m \u001b[43mdf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpivot\u001b[49m\u001b[43m(\u001b[49m\u001b[43mindex\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mindex\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcolumns\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mnum_duplicates\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mvalues\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mduplicates_prob\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/.pyenv/versions/3.12.2/envs/datatrove/lib/python3.12/site-packages/pandas/core/frame.py:9326\u001b[0m, in \u001b[0;36mDataFrame.pivot\u001b[0;34m(self, columns, index, values)\u001b[0m\n\u001b[1;32m   9319\u001b[0m \u001b[38;5;129m@Substitution\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m   9320\u001b[0m \u001b[38;5;129m@Appender\u001b[39m(_shared_docs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpivot\u001b[39m\u001b[38;5;124m\"\u001b[39m])\n\u001b[1;32m   9321\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mpivot\u001b[39m(\n\u001b[1;32m   9322\u001b[0m     \u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39m, columns, index\u001b[38;5;241m=\u001b[39mlib\u001b[38;5;241m.\u001b[39mno_default, values\u001b[38;5;241m=\u001b[39mlib\u001b[38;5;241m.\u001b[39mno_default\n\u001b[1;32m   9323\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m DataFrame:\n\u001b[1;32m   9324\u001b[0m     \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcore\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mreshape\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpivot\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m pivot\n\u001b[0;32m-> 9326\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mpivot\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mindex\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mindex\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcolumns\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcolumns\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mvalues\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mvalues\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/.pyenv/versions/3.12.2/envs/datatrove/lib/python3.12/site-packages/pandas/core/reshape/pivot.py:553\u001b[0m, in \u001b[0;36mpivot\u001b[0;34m(data, columns, index, values)\u001b[0m\n\u001b[1;32m    549\u001b[0m         index_list \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m    550\u001b[0m             data\u001b[38;5;241m.\u001b[39m_constructor_sliced(data\u001b[38;5;241m.\u001b[39mindex, name\u001b[38;5;241m=\u001b[39mdata\u001b[38;5;241m.\u001b[39mindex\u001b[38;5;241m.\u001b[39mname)\n\u001b[1;32m    551\u001b[0m         ]\n\u001b[1;32m    552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 553\u001b[0m     index_list \u001b[38;5;241m=\u001b[39m [\u001b[43mdata\u001b[49m\u001b[43m[\u001b[49m\u001b[43midx\u001b[49m\u001b[43m]\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m idx \u001b[38;5;129;01min\u001b[39;00m com\u001b[38;5;241m.\u001b[39mconvert_to_list_like(index)]\n\u001b[1;32m    555\u001b[0m data_columns \u001b[38;5;241m=\u001b[39m [data[col] \u001b[38;5;28;01mfor\u001b[39;00m col \u001b[38;5;129;01min\u001b[39;00m columns_listlike]\n\u001b[1;32m    556\u001b[0m index_list\u001b[38;5;241m.\u001b[39mextend(data_columns)\n",
+      "File \u001b[0;32m~/.pyenv/versions/3.12.2/envs/datatrove/lib/python3.12/site-packages/pandas/core/frame.py:4090\u001b[0m, in \u001b[0;36mDataFrame.__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   4088\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcolumns\u001b[38;5;241m.\u001b[39mnlevels \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m1\u001b[39m:\n\u001b[1;32m   4089\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_getitem_multilevel(key)\n\u001b[0;32m-> 4090\u001b[0m indexer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcolumns\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_loc\u001b[49m\u001b[43m(\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   4091\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m is_integer(indexer):\n\u001b[1;32m   4092\u001b[0m     indexer \u001b[38;5;241m=\u001b[39m [indexer]\n",
+      "File \u001b[0;32m~/.pyenv/versions/3.12.2/envs/datatrove/lib/python3.12/site-packages/pandas/core/indexes/base.py:3812\u001b[0m, in \u001b[0;36mIndex.get_loc\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   3807\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(casted_key, \u001b[38;5;28mslice\u001b[39m) \u001b[38;5;129;01mor\u001b[39;00m (\n\u001b[1;32m   3808\u001b[0m         \u001b[38;5;28misinstance\u001b[39m(casted_key, abc\u001b[38;5;241m.\u001b[39mIterable)\n\u001b[1;32m   3809\u001b[0m         \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28many\u001b[39m(\u001b[38;5;28misinstance\u001b[39m(x, \u001b[38;5;28mslice\u001b[39m) \u001b[38;5;28;01mfor\u001b[39;00m x \u001b[38;5;129;01min\u001b[39;00m casted_key)\n\u001b[1;32m   3810\u001b[0m     ):\n\u001b[1;32m   3811\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m InvalidIndexError(key)\n\u001b[0;32m-> 3812\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m(key) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01merr\u001b[39;00m\n\u001b[1;32m   3813\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m:\n\u001b[1;32m   3814\u001b[0m     \u001b[38;5;66;03m# If we have a listlike key, _check_indexing_error will raise\u001b[39;00m\n\u001b[1;32m   3815\u001b[0m     \u001b[38;5;66;03m#  InvalidIndexError. Otherwise we fall through and re-raise\u001b[39;00m\n\u001b[1;32m   3816\u001b[0m     \u001b[38;5;66;03m#  the TypeError.\u001b[39;00m\n\u001b[1;32m   3817\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_check_indexing_error(key)\n",
+      "\u001b[0;31mKeyError\u001b[0m: 'index'"
+     ]
+    }
+   ],
+   "source": [
+    "# Take the sumarized_df and pivotdf"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.plotly.v1+json": {
+       "config": {
+        "plotlyServerURL": "https://plot.ly"
+       },
+       "data": [
+        {
+         "name": "1",
+         "type": "bar",
+         "x": [
+          "1B",
+          "10B",
+          "100B",
+          "350B",
+          "1T"
+         ],
+         "y": [
+          0.994974,
+          0.9515081,
+          0.60887281,
+          0.1741474885714285,
+          0.006232416
+         ]
+        },
+        {
+         "name": "2",
+         "type": "bar",
+         "x": [
+          "1B",
+          "10B",
+          "100B",
+          "350B",
+          "1T"
+         ],
+         "y": [
+          0.005008,
+          0.047331,
+          0.30282154,
+          0.3071204342857143,
+          0.032470074
+         ]
+        },
+        {
+         "name": "3",
+         "type": "bar",
+         "x": [
+          "1B",
+          "10B",
+          "100B",
+          "350B",
+          "1T"
+         ],
+         "y": [
+          0.000018,
+          0.0011439,
+          0.0745482,
+          0.2680183371428571,
+          0.083742993
+         ]
+        },
+        {
+         "name": "4-8",
+         "type": "bar",
+         "x": [
+          "1B",
+          "10B",
+          "100B",
+          "350B",
+          "1T"
+         ],
+         "y": [
+          0,
+          0.000017,
+          0.01375745,
+          0.25064894285714273,
+          0.8176358810000001
+         ]
+        },
+        {
+         "name": "8-16",
+         "type": "bar",
+         "x": [
+          "1B",
+          "10B",
+          "100B",
+          "350B",
+          "1T"
+         ],
+         "y": [
+          0,
+          0,
+          0,
+          0.00006479714285714286,
+          0.05991048400000001
+         ]
+        },
+        {
+         "name": "16-32",
+         "type": "bar",
+         "x": [
+          "1B",
+          "10B",
+          "100B",
+          "350B",
+          "1T"
+         ],
+         "y": [
+          0,
+          0,
+          0,
+          0,
+          0.000008152000000000001
+         ]
+        }
+       ],
+       "layout": {
+        "barmode": "stack",
+        "legend": {
+         "title": {
+          "text": "# duplicates"
+         }
+        },
+        "template": {
+         "data": {
+          "bar": [
+           {
+            "error_x": {
+             "color": "#2a3f5f"
+            },
+            "error_y": {
+             "color": "#2a3f5f"
+            },
+            "marker": {
+             "line": {
+              "color": "#E5ECF6",
+              "width": 0.5
+             },
+             "pattern": {
+              "fillmode": "overlay",
+              "size": 10,
+              "solidity": 0.2
+             }
+            },
+            "type": "bar"
+           }
+          ],
+          "barpolar": [
+           {
+            "marker": {
+             "line": {
+              "color": "#E5ECF6",
+              "width": 0.5
+             },
+             "pattern": {
+              "fillmode": "overlay",
+              "size": 10,
+              "solidity": 0.2
+             }
+            },
+            "type": "barpolar"
+           }
+          ],
+          "carpet": [
+           {
+            "aaxis": {
+             "endlinecolor": "#2a3f5f",
+             "gridcolor": "white",
+             "linecolor": "white",
+             "minorgridcolor": "white",
+             "startlinecolor": "#2a3f5f"
+            },
+            "baxis": {
+             "endlinecolor": "#2a3f5f",
+             "gridcolor": "white",
+             "linecolor": "white",
+             "minorgridcolor": "white",
+             "startlinecolor": "#2a3f5f"
+            },
+            "type": "carpet"
+           }
+          ],
+          "choropleth": [
+           {
+            "colorbar": {
+             "outlinewidth": 0,
+             "ticks": ""
+            },
+            "type": "choropleth"
+           }
+          ],
+          "contour": [
+           {
+            "colorbar": {
+             "outlinewidth": 0,
+             "ticks": ""
+            },
+            "colorscale": [
+             [
+              0,
+              "#0d0887"
+             ],
+             [
+              0.1111111111111111,
+              "#46039f"
+             ],
+             [
+              0.2222222222222222,
+              "#7201a8"
+             ],
+             [
+              0.3333333333333333,
+              "#9c179e"
+             ],
+             [
+              0.4444444444444444,
+              "#bd3786"
+             ],
+             [
+              0.5555555555555556,
+              "#d8576b"
+             ],
+             [
+              0.6666666666666666,
+              "#ed7953"
+             ],
+             [
+              0.7777777777777778,
+              "#fb9f3a"
+             ],
+             [
+              0.8888888888888888,
+              "#fdca26"
+             ],
+             [
+              1,
+              "#f0f921"
+             ]
+            ],
+            "type": "contour"
+           }
+          ],
+          "contourcarpet": [
+           {
+            "colorbar": {
+             "outlinewidth": 0,
+             "ticks": ""
+            },
+            "type": "contourcarpet"
+           }
+          ],
+          "heatmap": [
+           {
+            "colorbar": {
+             "outlinewidth": 0,
+             "ticks": ""
+            },
+            "colorscale": [
+             [
+              0,
+              "#0d0887"
+             ],
+             [
+              0.1111111111111111,
+              "#46039f"
+             ],
+             [
+              0.2222222222222222,
+              "#7201a8"
+             ],
+             [
+              0.3333333333333333,
+              "#9c179e"
+             ],
+             [
+              0.4444444444444444,
+              "#bd3786"
+             ],
+             [
+              0.5555555555555556,
+              "#d8576b"
+             ],
+             [
+              0.6666666666666666,
+              "#ed7953"
+             ],
+             [
+              0.7777777777777778,
+              "#fb9f3a"
+             ],
+             [
+              0.8888888888888888,
+              "#fdca26"
+             ],
+             [
+              1,
+              "#f0f921"
+             ]
+            ],
+            "type": "heatmap"
+           }
+          ],
+          "heatmapgl": [
+           {
+            "colorbar": {
+             "outlinewidth": 0,
+             "ticks": ""
+            },
+            "colorscale": [
+             [
+              0,
+              "#0d0887"
+             ],
+             [
+              0.1111111111111111,
+              "#46039f"
+             ],
+             [
+              0.2222222222222222,
+              "#7201a8"
+             ],
+             [
+              0.3333333333333333,
+              "#9c179e"
+             ],
+             [
+              0.4444444444444444,
+              "#bd3786"
+             ],
+             [
+              0.5555555555555556,
+              "#d8576b"
+             ],
+             [
+              0.6666666666666666,
+              "#ed7953"
+             ],
+             [
+              0.7777777777777778,
+              "#fb9f3a"
+             ],
+             [
+              0.8888888888888888,
+              "#fdca26"
+             ],
+             [
+              1,
+              "#f0f921"
+             ]
+            ],
+            "type": "heatmapgl"
+           }
+          ],
+          "histogram": [
+           {
+            "marker": {
+             "pattern": {
+              "fillmode": "overlay",
+              "size": 10,
+              "solidity": 0.2
+             }
+            },
+            "type": "histogram"
+           }
+          ],
+          "histogram2d": [
+           {
+            "colorbar": {
+             "outlinewidth": 0,
+             "ticks": ""
+            },
+            "colorscale": [
+             [
+              0,
+              "#0d0887"
+             ],
+             [
+              0.1111111111111111,
+              "#46039f"
+             ],
+             [
+              0.2222222222222222,
+              "#7201a8"
+             ],
+             [
+              0.3333333333333333,
+              "#9c179e"
+             ],
+             [
+              0.4444444444444444,
+              "#bd3786"
+             ],
+             [
+              0.5555555555555556,
+              "#d8576b"
+             ],
+             [
+              0.6666666666666666,
+              "#ed7953"
+             ],
+             [
+              0.7777777777777778,
+              "#fb9f3a"
+             ],
+             [
+              0.8888888888888888,
+              "#fdca26"
+             ],
+             [
+              1,
+              "#f0f921"
+             ]
+            ],
+            "type": "histogram2d"
+           }
+          ],
+          "histogram2dcontour": [
+           {
+            "colorbar": {
+             "outlinewidth": 0,
+             "ticks": ""
+            },
+            "colorscale": [
+             [
+              0,
+              "#0d0887"
+             ],
+             [
+              0.1111111111111111,
+              "#46039f"
+             ],
+             [
+              0.2222222222222222,
+              "#7201a8"
+             ],
+             [
+              0.3333333333333333,
+              "#9c179e"
+             ],
+             [
+              0.4444444444444444,
+              "#bd3786"
+             ],
+             [
+              0.5555555555555556,
+              "#d8576b"
+             ],
+             [
+              0.6666666666666666,
+              "#ed7953"
+             ],
+             [
+              0.7777777777777778,
+              "#fb9f3a"
+             ],
+             [
+              0.8888888888888888,
+              "#fdca26"
+             ],
+             [
+              1,
+              "#f0f921"
+             ]
+            ],
+            "type": "histogram2dcontour"
+           }
+          ],
+          "mesh3d": [
+           {
+            "colorbar": {
+             "outlinewidth": 0,
+             "ticks": ""
+            },
+            "type": "mesh3d"
+           }
+          ],
+          "parcoords": [
+           {
+            "line": {
+             "colorbar": {
+              "outlinewidth": 0,
+              "ticks": ""
+             }
+            },
+            "type": "parcoords"
+           }
+          ],
+          "pie": [
+           {
+            "automargin": true,
+            "type": "pie"
+           }
+          ],
+          "scatter": [
+           {
+            "fillpattern": {
+             "fillmode": "overlay",
+             "size": 10,
+             "solidity": 0.2
+            },
+            "type": "scatter"
+           }
+          ],
+          "scatter3d": [
+           {
+            "line": {
+             "colorbar": {
+              "outlinewidth": 0,
+              "ticks": ""
+             }
+            },
+            "marker": {
+             "colorbar": {
+              "outlinewidth": 0,
+              "ticks": ""
+             }
+            },
+            "type": "scatter3d"
+           }
+          ],
+          "scattercarpet": [
+           {
+            "marker": {
+             "colorbar": {
+              "outlinewidth": 0,
+              "ticks": ""
+             }
+            },
+            "type": "scattercarpet"
+           }
+          ],
+          "scattergeo": [
+           {
+            "marker": {
+             "colorbar": {
+              "outlinewidth": 0,
+              "ticks": ""
+             }
+            },
+            "type": "scattergeo"
+           }
+          ],
+          "scattergl": [
+           {
+            "marker": {
+             "colorbar": {
+              "outlinewidth": 0,
+              "ticks": ""
+             }
+            },
+            "type": "scattergl"
+           }
+          ],
+          "scattermapbox": [
+           {
+            "marker": {
+             "colorbar": {
+              "outlinewidth": 0,
+              "ticks": ""
+             }
+            },
+            "type": "scattermapbox"
+           }
+          ],
+          "scatterpolar": [
+           {
+            "marker": {
+             "colorbar": {
+              "outlinewidth": 0,
+              "ticks": ""
+             }
+            },
+            "type": "scatterpolar"
+           }
+          ],
+          "scatterpolargl": [
+           {
+            "marker": {
+             "colorbar": {
+              "outlinewidth": 0,
+              "ticks": ""
+             }
+            },
+            "type": "scatterpolargl"
+           }
+          ],
+          "scatterternary": [
+           {
+            "marker": {
+             "colorbar": {
+              "outlinewidth": 0,
+              "ticks": ""
+             }
+            },
+            "type": "scatterternary"
+           }
+          ],
+          "surface": [
+           {
+            "colorbar": {
+             "outlinewidth": 0,
+             "ticks": ""
+            },
+            "colorscale": [
+             [
+              0,
+              "#0d0887"
+             ],
+             [
+              0.1111111111111111,
+              "#46039f"
+             ],
+             [
+              0.2222222222222222,
+              "#7201a8"
+             ],
+             [
+              0.3333333333333333,
+              "#9c179e"
+             ],
+             [
+              0.4444444444444444,
+              "#bd3786"
+             ],
+             [
+              0.5555555555555556,
+              "#d8576b"
+             ],
+             [
+              0.6666666666666666,
+              "#ed7953"
+             ],
+             [
+              0.7777777777777778,
+              "#fb9f3a"
+             ],
+             [
+              0.8888888888888888,
+              "#fdca26"
+             ],
+             [
+              1,
+              "#f0f921"
+             ]
+            ],
+            "type": "surface"
+           }
+          ],
+          "table": [
+           {
+            "cells": {
+             "fill": {
+              "color": "#EBF0F8"
+             },
+             "line": {
+              "color": "white"
+             }
+            },
+            "header": {
+             "fill": {
+              "color": "#C8D4E3"
+             },
+             "line": {
+              "color": "white"
+             }
+            },
+            "type": "table"
+           }
+          ]
+         },
+         "layout": {
+          "annotationdefaults": {
+           "arrowcolor": "#2a3f5f",
+           "arrowhead": 0,
+           "arrowwidth": 1
+          },
+          "autotypenumbers": "strict",
+          "coloraxis": {
+           "colorbar": {
+            "outlinewidth": 0,
+            "ticks": ""
+           }
+          },
+          "colorscale": {
+           "diverging": [
+            [
+             0,
+             "#8e0152"
+            ],
+            [
+             0.1,
+             "#c51b7d"
+            ],
+            [
+             0.2,
+             "#de77ae"
+            ],
+            [
+             0.3,
+             "#f1b6da"
+            ],
+            [
+             0.4,
+             "#fde0ef"
+            ],
+            [
+             0.5,
+             "#f7f7f7"
+            ],
+            [
+             0.6,
+             "#e6f5d0"
+            ],
+            [
+             0.7,
+             "#b8e186"
+            ],
+            [
+             0.8,
+             "#7fbc41"
+            ],
+            [
+             0.9,
+             "#4d9221"
+            ],
+            [
+             1,
+             "#276419"
+            ]
+           ],
+           "sequential": [
+            [
+             0,
+             "#0d0887"
+            ],
+            [
+             0.1111111111111111,
+             "#46039f"
+            ],
+            [
+             0.2222222222222222,
+             "#7201a8"
+            ],
+            [
+             0.3333333333333333,
+             "#9c179e"
+            ],
+            [
+             0.4444444444444444,
+             "#bd3786"
+            ],
+            [
+             0.5555555555555556,
+             "#d8576b"
+            ],
+            [
+             0.6666666666666666,
+             "#ed7953"
+            ],
+            [
+             0.7777777777777778,
+             "#fb9f3a"
+            ],
+            [
+             0.8888888888888888,
+             "#fdca26"
+            ],
+            [
+             1,
+             "#f0f921"
+            ]
+           ],
+           "sequentialminus": [
+            [
+             0,
+             "#0d0887"
+            ],
+            [
+             0.1111111111111111,
+             "#46039f"
+            ],
+            [
+             0.2222222222222222,
+             "#7201a8"
+            ],
+            [
+             0.3333333333333333,
+             "#9c179e"
+            ],
+            [
+             0.4444444444444444,
+             "#bd3786"
+            ],
+            [
+             0.5555555555555556,
+             "#d8576b"
+            ],
+            [
+             0.6666666666666666,
+             "#ed7953"
+            ],
+            [
+             0.7777777777777778,
+             "#fb9f3a"
+            ],
+            [
+             0.8888888888888888,
+             "#fdca26"
+            ],
+            [
+             1,
+             "#f0f921"
+            ]
+           ]
+          },
+          "colorway": [
+           "#636efa",
+           "#EF553B",
+           "#00cc96",
+           "#ab63fa",
+           "#FFA15A",
+           "#19d3f3",
+           "#FF6692",
+           "#B6E880",
+           "#FF97FF",
+           "#FECB52"
+          ],
+          "font": {
+           "color": "#2a3f5f"
+          },
+          "geo": {
+           "bgcolor": "white",
+           "lakecolor": "white",
+           "landcolor": "#E5ECF6",
+           "showlakes": true,
+           "showland": true,
+           "subunitcolor": "white"
+          },
+          "hoverlabel": {
+           "align": "left"
+          },
+          "hovermode": "closest",
+          "mapbox": {
+           "style": "light"
+          },
+          "paper_bgcolor": "white",
+          "plot_bgcolor": "#E5ECF6",
+          "polar": {
+           "angularaxis": {
+            "gridcolor": "white",
+            "linecolor": "white",
+            "ticks": ""
+           },
+           "bgcolor": "#E5ECF6",
+           "radialaxis": {
+            "gridcolor": "white",
+            "linecolor": "white",
+            "ticks": ""
+           }
+          },
+          "scene": {
+           "xaxis": {
+            "backgroundcolor": "#E5ECF6",
+            "gridcolor": "white",
+            "gridwidth": 2,
+            "linecolor": "white",
+            "showbackground": true,
+            "ticks": "",
+            "zerolinecolor": "white"
+           },
+           "yaxis": {
+            "backgroundcolor": "#E5ECF6",
+            "gridcolor": "white",
+            "gridwidth": 2,
+            "linecolor": "white",
+            "showbackground": true,
+            "ticks": "",
+            "zerolinecolor": "white"
+           },
+           "zaxis": {
+            "backgroundcolor": "#E5ECF6",
+            "gridcolor": "white",
+            "gridwidth": 2,
+            "linecolor": "white",
+            "showbackground": true,
+            "ticks": "",
+            "zerolinecolor": "white"
+           }
+          },
+          "shapedefaults": {
+           "line": {
+            "color": "#2a3f5f"
+           }
+          },
+          "ternary": {
+           "aaxis": {
+            "gridcolor": "white",
+            "linecolor": "white",
+            "ticks": ""
+           },
+           "baxis": {
+            "gridcolor": "white",
+            "linecolor": "white",
+            "ticks": ""
+           },
+           "bgcolor": "#E5ECF6",
+           "caxis": {
+            "gridcolor": "white",
+            "linecolor": "white",
+            "ticks": ""
+           }
+          },
+          "title": {
+           "x": 0.05
+          },
+          "xaxis": {
+           "automargin": true,
+           "gridcolor": "white",
+           "linecolor": "white",
+           "ticks": "",
+           "title": {
+            "standoff": 15
+           },
+           "zerolinecolor": "white",
+           "zerolinewidth": 2
+          },
+          "yaxis": {
+           "automargin": true,
+           "gridcolor": "white",
+           "linecolor": "white",
+           "ticks": "",
+           "title": {
+            "standoff": 15
+           },
+           "zerolinecolor": "white",
+           "zerolinewidth": 2
+          }
+         }
+        },
+        "title": {
+         "text": "Sampling from 100 identical buckets with 200B tokens each"
+        },
+        "xaxis": {
+         "title": {
+          "text": "Sample size"
+         }
+        },
+        "yaxis": {
+         "range": [
+          0,
+          1.000001
+         ],
+         "title": {
+          "text": "Dataset fraction"
+         }
+        }
+       }
+      }
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "import matplotlib.pyplot as plt\n",
+    "\n",
+    "import plotly.graph_objects as go\n",
+    "\n",
+    "df = pd.read_csv(\"./data/duplicates-simulation.csv\", index_col=0)\n",
+    "\n",
+    "def summarize_ranges(df):\n",
+    "    df_summarized = pd.DataFrame(index=['1', '2', '3', '4-8', '8-16', '16-32'], columns=df.columns)\n",
+    "    df_summarized.loc['1'] = df.iloc[0]\n",
+    "    df_summarized.loc['2'] = df.iloc[1]\n",
+    "    df_summarized.loc['3'] = df.iloc[2]\n",
+    "    df_summarized.loc['4-8'] = df.iloc[3:9].sum()\n",
+    "    df_summarized.loc['8-16'] = df.iloc[9:17].sum()\n",
+    "    df_summarized.loc['16-32'] = df.iloc[17:].sum()\n",
+    "    return df_summarized\n",
+    "\n",
+    "summarized_df = summarize_ranges(df).T\n",
+    "\n",
+    "# Create a stacked bar chart using Plotly\n",
+    "fig = go.Figure()\n",
+    "for col in summarized_df.columns:\n",
+    "    fig.add_trace(go.Bar(\n",
+    "        x=summarized_df.index,\n",
+    "        y=summarized_df[col],\n",
+    "        name=col\n",
+    "    ))\n",
+    "\n",
+    "fig.update_layout(\n",
+    "    barmode='stack',\n",
+    "    title_text=\"Sampling from 100 identical buckets with 200B tokens each\",\n",
+    "    xaxis_title=\"Sample size\",\n",
+    "    yaxis_title=\"Dataset fraction\",\n",
+    "    yaxis=dict(range=[0, 1.000001]),\n",
+    "    legend_title=\"# duplicates\",\n",
+    ")\n",
+    "\n",
+    "fig.show()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Index(['1B', '10B', '100B', '350B', '1T'], dtype='object')"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "summarized_df.index"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "datatrove",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

notebooks/plot_histograms_cross.ipynb

@@ -0,0 +1,155 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from collections import defaultdict\n",
+    "\n",
+    "import pandas as pd\n",
+    "\n",
+    "\n",
+    "def get_setting(name):\n",
+    "    if \"terminal-punct\" in name:\n",
+    "        return {\"x\": \"Fraction of lines ended with punctuation\", \"ylim\": (0, 0.1)}\n",
+    "    \n",
+    "    if \"line-dedup\" in name:\n",
+    "        return {\"x\": \"Fraction of chars in duplicated lines\", \"xlim\": (0, 0.1), \"ylim\": (0,0.02)}\n",
+    "    \n",
+    "    if \"short-line\" in name:\n",
+    "        return {\"x\": \"Fraction of lines shorter than 30 chars\", \"xlim\": (0.4, 1.0), \"ylim\": (0,0.05)}\n",
+    "    \n",
+    "    if \"avg_words_per_line\" in name:\n",
+    "        return {\"x\": \"Avg. words per line\", \"x-log\": True, \"x-log\": True, \"round\": 0}\n",
+    "    if \"avg_line_length\" in name:\n",
+    "        return {\"x\": \"Avg. words per line\", \"x-log\": True, \"round\": 0}\n",
+    "    \n",
+    "    if \"global-length.json\" == name:\n",
+    "        return {\"x\": \"Num. UTF-8 chars\", \"x-log\": True}\n",
+    "    \n",
+    "    if \"global-digit_ratio.json\" == name:\n",
+    "        return {\"x\": \"Digit ratio\", \"xlim\": (0, 0.25)}\n",
+    "    \n",
+    "    if \"global-avg_word_length.json\" == name:\n",
+    "        return {\"x\": \"Avg. word length\", \"xlim\": (2.5, 6.5)}\n",
+    "\n",
+    "    \n",
+    "    raise ValueError(f\"Unknown dataset name: {name}\")\n",
+    "\n",
+    "\n",
+    "def plot_scatter(data):\n",
+    "    \"\"\"\n",
+    "    Plot scatter plots with smoothing for each dataset in the data list on a single grid.\n",
+    "    Each dataset is expected to be a dictionary with the first key as the dataset name,\n",
+    "    and the value as another dictionary where keys are data points and values are their counts.\n",
+    "    \"\"\"\n",
+    "    import matplotlib.pyplot as plt\n",
+    "    import numpy as np\n",
+    "\n",
+    "    # Determine the number of plots and create a subplot grid\n",
+    "    num_datasets = len(data)\n",
+    "    cols = 2  # Define number of columns in the grid\n",
+    "    rows = (num_datasets) // cols  # Calculate the required number of rows\n",
+    "    fig, axs = plt.subplots(rows, cols, figsize=(8 * cols, 3 * rows), dpi=350)\n",
+    "    if rows * cols > 1:\n",
+    "        axs = axs.flatten()  # Flatten the array of axes if more than one subplot\n",
+    "    else:\n",
+    "        axs = [axs]  # Encapsulate the single AxesSubplot object into a list for uniform handling\n",
+    "\n",
+    "    plot_index = 0\n",
+    "    legend_handles = []  # List to store handles for the legend\n",
+    "    legend_labels = []  # List to store labels for the legend\n",
+    "    for name, dataset in data.items():\n",
+    "        setting = get_setting(name)\n",
+    "        ax = axs[plot_index]\n",
+    "        if \"name\" in setting:\n",
+    "            ax.set_title(setting[\"name\"])\n",
+    "        if \"x\" in setting:\n",
+    "            ax.set_xlabel(setting[\"x\"])\n",
+    "        if \"xlim\" in setting:\n",
+    "            ax.set_xlim(setting[\"xlim\"])\n",
+    "        if \"ylim\" in setting:\n",
+    "            ax.set_ylim(setting[\"ylim\"])\n",
+    "        if \"x-log\" in setting:\n",
+    "            ax.set_xscale('log')\n",
+    "\n",
+    "        # Use 2 decimal places for the y-axis labels\n",
+    "        ax.yaxis.set_major_formatter('{x:.3f}')\n",
+    "\n",
+    "\n",
+    "        plot_index += 1\n",
+    "        # Each dataset may contain multiple lines\n",
+    "        for i, (line_name, line_data) in enumerate(dataset.items()):\n",
+    "            if \"round\" in setting:\n",
+    "                tmp_line_data = defaultdict(list)\n",
+    "                for p, p_v in line_data.items():\n",
+    "                    rounded_key = str(round(float(p), setting[\"round\"]))\n",
+    "                    tmp_line_data[rounded_key].append(p_v)\n",
+    "\n",
+    "                # If you want to sum the values that have the same rounded key\n",
+    "                tmp_line_data = {k: sum(v) for k, v in tmp_line_data.items()}\n",
+    "                line_data = tmp_line_data\n",
+    "            \n",
+    "            # Check that if you sum the values you get 1\n",
+    "            assert sum(line_data.values()) == 1\n",
+    "\n",
+    "            # Add smoothing for 4-5 points\n",
+    "            # Implementing smoothing using a rolling window\n",
+    "            line_name = rename_dataset(line_name)\n",
+    "            # Sorting the line data by keys\n",
+    "            sorted_line_data = dict(sorted(line_data.items(), key=lambda item: float(item[0])))\n",
+    "\n",
+    "            window_size = setting.get(\"window_size\", 5)  # Define the window size for smoothing\n",
+    "            x = np.array(list(sorted_line_data.keys()), dtype=float)\n",
+    "            y = np.array(list(sorted_line_data.values()), dtype=float)\n",
+    "            if len(y) >= window_size:  # Ensure there are enough points to apply smoothing\n",
+    "                # Convert y to a pandas Series to use rolling function\n",
+    "                y_series = pd.Series(y)\n",
+    "                # Apply rolling window and mean to smooth the data\n",
+    "                y_smoothed = y_series.rolling(window=window_size).mean()\n",
+    "                # Drop NaN values that result from the rolling mean calculation\n",
+    "                y_smoothed = y_smoothed.dropna()\n",
+    "                # Update x to correspond to the length of the smoothed y\n",
+    "                x = x[len(x) - len(y_smoothed):]\n",
+    "                y = y_smoothed.to_numpy()  # Convert back to numpy array for plotting\n",
+    "\n",
+    "\n",
+    "\n",
+    "            # Use the line name as the label to unify same line names across different plots\n",
+    "\n",
+    "            line, = ax.plot(x, y, label=line_name)  # Use default colors\n",
+    "            if line_name not in legend_labels:\n",
+    "                legend_handles.append(line)\n",
+    "                legend_labels.append(line_name)\n",
+    "\n",
+    "    # Place a single shared legend on the top of the figure\n",
+    "    fig.legend(handles=legend_handles, labels=legend_labels, loc='lower center', ncol=1)\n",
+    "    for ax in axs:\n",
+    "        ax.set_ylabel('Document Frequency')\n",
+    "\n",
+    "    fig.suptitle(\"Histograms of selected statistics\")\n",
+    "    plt.tight_layout(rect=[0, 0.15, 1, 1])  # Adjust the layout to make room for the legend\n",
+    "    fig.set_size_inches(13, 6)  # Set the figure size to 18 inches by 12 inches\n",
+    "    plt.show()\n",
+    "\n",
+    "plot_scatter(data)\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

notebooks/plot_removed_data_dedup.ipynb

@@ -0,0 +1,1578 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "138889b92720ce2e",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-04-30T13:28:07.130909Z",
+     "start_time": "2024-04-30T13:28:06.470042Z"
+    },
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>runname</th>\n",
+       "      <th>seed</th>\n",
+       "      <th>steps</th>\n",
+       "      <th>agg_score</th>\n",
+       "      <th>commonsense_qa/acc</th>\n",
+       "      <th>commonsense_qa/acc_norm</th>\n",
+       "      <th>hellaswag/acc</th>\n",
+       "      <th>hellaswag/acc_norm</th>\n",
+       "      <th>openbookqa/acc</th>\n",
+       "      <th>openbookqa/acc_norm</th>\n",
+       "      <th>...</th>\n",
+       "      <th>siqa/acc</th>\n",
+       "      <th>siqa/acc_norm</th>\n",
+       "      <th>winogrande/acc</th>\n",
+       "      <th>winogrande/acc_norm</th>\n",
+       "      <th>sciq/acc</th>\n",
+       "      <th>sciq/acc_norm</th>\n",
+       "      <th>arc/acc</th>\n",
+       "      <th>arc/acc_norm</th>\n",
+       "      <th>mmlu/acc</th>\n",
+       "      <th>mmlu/acc_norm</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>deduped_removed_cross</td>\n",
+       "      <td>5</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.330893</td>\n",
+       "      <td>0.186</td>\n",
+       "      <td>0.233</td>\n",
+       "      <td>0.272</td>\n",
+       "      <td>0.258</td>\n",
+       "      <td>0.166</td>\n",
+       "      <td>0.286</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.367</td>\n",
+       "      <td>0.362</td>\n",
+       "      <td>0.516</td>\n",
+       "      <td>0.497</td>\n",
+       "      <td>0.208</td>\n",
+       "      <td>0.202</td>\n",
+       "      <td>0.2195</td>\n",
+       "      <td>0.2510</td>\n",
+       "      <td>0.230294</td>\n",
+       "      <td>0.250147</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>deduped_removed_cross</td>\n",
+       "      <td>5</td>\n",
+       "      <td>1000</td>\n",
+       "      <td>0.354090</td>\n",
+       "      <td>0.253</td>\n",
+       "      <td>0.257</td>\n",
+       "      <td>0.290</td>\n",
+       "      <td>0.278</td>\n",
+       "      <td>0.124</td>\n",
+       "      <td>0.264</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.368</td>\n",
+       "      <td>0.389</td>\n",
+       "      <td>0.509</td>\n",
+       "      <td>0.491</td>\n",
+       "      <td>0.582</td>\n",
+       "      <td>0.516</td>\n",
+       "      <td>0.2825</td>\n",
+       "      <td>0.2955</td>\n",
+       "      <td>0.239520</td>\n",
+       "      <td>0.253223</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>deduped_removed_cross</td>\n",
+       "      <td>5</td>\n",
+       "      <td>2000</td>\n",
+       "      <td>0.373601</td>\n",
+       "      <td>0.274</td>\n",
+       "      <td>0.290</td>\n",
+       "      <td>0.313</td>\n",
+       "      <td>0.312</td>\n",
+       "      <td>0.116</td>\n",
+       "      <td>0.258</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.367</td>\n",
+       "      <td>0.397</td>\n",
+       "      <td>0.516</td>\n",
+       "      <td>0.505</td>\n",
+       "      <td>0.686</td>\n",
+       "      <td>0.582</td>\n",
+       "      <td>0.3090</td>\n",
+       "      <td>0.3200</td>\n",
+       "      <td>0.247320</td>\n",
+       "      <td>0.262812</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>deduped_removed_cross</td>\n",
+       "      <td>5</td>\n",
+       "      <td>3000</td>\n",
+       "      <td>0.383122</td>\n",
+       "      <td>0.306</td>\n",
+       "      <td>0.292</td>\n",
+       "      <td>0.323</td>\n",
+       "      <td>0.335</td>\n",
+       "      <td>0.150</td>\n",
+       "      <td>0.278</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.371</td>\n",
+       "      <td>0.401</td>\n",
+       "      <td>0.513</td>\n",
+       "      <td>0.500</td>\n",
+       "      <td>0.712</td>\n",
+       "      <td>0.611</td>\n",
+       "      <td>0.3075</td>\n",
+       "      <td>0.3415</td>\n",
+       "      <td>0.248568</td>\n",
+       "      <td>0.263474</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>deduped_removed_cross</td>\n",
+       "      <td>5</td>\n",
+       "      <td>4000</td>\n",
+       "      <td>0.390222</td>\n",
+       "      <td>0.300</td>\n",
+       "      <td>0.292</td>\n",
+       "      <td>0.324</td>\n",
+       "      <td>0.351</td>\n",
+       "      <td>0.144</td>\n",
+       "      <td>0.278</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.386</td>\n",
+       "      <td>0.395</td>\n",
+       "      <td>0.511</td>\n",
+       "      <td>0.511</td>\n",
+       "      <td>0.750</td>\n",
+       "      <td>0.658</td>\n",
+       "      <td>0.3260</td>\n",
+       "      <td>0.3445</td>\n",
+       "      <td>0.259246</td>\n",
+       "      <td>0.273276</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>deduped_removed_cross</td>\n",
+       "      <td>5</td>\n",
+       "      <td>5000</td>\n",
+       "      <td>0.400239</td>\n",
+       "      <td>0.322</td>\n",
+       "      <td>0.308</td>\n",
+       "      <td>0.325</td>\n",
+       "      <td>0.364</td>\n",
+       "      <td>0.172</td>\n",
+       "      <td>0.298</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.382</td>\n",
+       "      <td>0.398</td>\n",
+       "      <td>0.518</td>\n",
+       "      <td>0.522</td>\n",
+       "      <td>0.751</td>\n",
+       "      <td>0.661</td>\n",
+       "      <td>0.3470</td>\n",
+       "      <td>0.3545</td>\n",
+       "      <td>0.258485</td>\n",
+       "      <td>0.271414</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>deduped_removed_cross</td>\n",
+       "      <td>5</td>\n",
+       "      <td>6000</td>\n",
+       "      <td>0.401484</td>\n",
+       "      <td>0.315</td>\n",
+       "      <td>0.314</td>\n",
+       "      <td>0.341</td>\n",
+       "      <td>0.372</td>\n",
+       "      <td>0.162</td>\n",
+       "      <td>0.314</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.377</td>\n",
+       "      <td>0.390</td>\n",
+       "      <td>0.498</td>\n",
+       "      <td>0.492</td>\n",
+       "      <td>0.776</td>\n",
+       "      <td>0.669</td>\n",
+       "      <td>0.3530</td>\n",
+       "      <td>0.3565</td>\n",
+       "      <td>0.261842</td>\n",
+       "      <td>0.276371</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>deduped_removed_cross</td>\n",
+       "      <td>5</td>\n",
+       "      <td>7000</td>\n",
+       "      <td>0.403533</td>\n",
+       "      <td>0.324</td>\n",
+       "      <td>0.315</td>\n",
+       "      <td>0.350</td>\n",
+       "      <td>0.386</td>\n",
+       "      <td>0.188</td>\n",
+       "      <td>0.298</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.376</td>\n",
+       "      <td>0.384</td>\n",
+       "      <td>0.518</td>\n",
+       "      <td>0.521</td>\n",
+       "      <td>0.769</td>\n",
+       "      <td>0.672</td>\n",
+       "      <td>0.3625</td>\n",
+       "      <td>0.3585</td>\n",
+       "      <td>0.265558</td>\n",
+       "      <td>0.274768</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>deduped_removed_cross</td>\n",
+       "      <td>5</td>\n",
+       "      <td>8000</td>\n",
+       "      <td>0.411774</td>\n",
+       "      <td>0.344</td>\n",
+       "      <td>0.313</td>\n",
+       "      <td>0.352</td>\n",
+       "      <td>0.409</td>\n",
+       "      <td>0.170</td>\n",
+       "      <td>0.310</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.374</td>\n",
+       "      <td>0.391</td>\n",
+       "      <td>0.530</td>\n",
+       "      <td>0.521</td>\n",
+       "      <td>0.781</td>\n",
+       "      <td>0.677</td>\n",
+       "      <td>0.3530</td>\n",
+       "      <td>0.3615</td>\n",
+       "      <td>0.267141</td>\n",
+       "      <td>0.283691</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
+       "      <td>deduped_removed_cross</td>\n",
+       "      <td>5</td>\n",
+       "      <td>9000</td>\n",
+       "      <td>0.410993</td>\n",
+       "      <td>0.335</td>\n",
+       "      <td>0.322</td>\n",
+       "      <td>0.361</td>\n",
+       "      <td>0.404</td>\n",
+       "      <td>0.182</td>\n",
+       "      <td>0.294</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.374</td>\n",
+       "      <td>0.391</td>\n",
+       "      <td>0.526</td>\n",
+       "      <td>0.514</td>\n",
+       "      <td>0.769</td>\n",
+       "      <td>0.672</td>\n",
+       "      <td>0.3630</td>\n",
+       "      <td>0.3715</td>\n",
+       "      <td>0.266464</td>\n",
+       "      <td>0.284446</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>10</th>\n",
+       "      <td>deduped_removed_cross</td>\n",
+       "      <td>5</td>\n",
+       "      <td>10000</td>\n",
+       "      <td>0.417883</td>\n",
+       "      <td>0.330</td>\n",
+       "      <td>0.320</td>\n",
+       "      <td>0.370</td>\n",
+       "      <td>0.417</td>\n",
+       "      <td>0.192</td>\n",
+       "      <td>0.324</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.389</td>\n",
+       "      <td>0.389</td>\n",
+       "      <td>0.518</td>\n",
+       "      <td>0.524</td>\n",
+       "      <td>0.785</td>\n",
+       "      <td>0.682</td>\n",
+       "      <td>0.3735</td>\n",
+       "      <td>0.3745</td>\n",
+       "      <td>0.268085</td>\n",
+       "      <td>0.283562</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>11</th>\n",
+       "      <td>deduped_removed_cross</td>\n",
+       "      <td>5</td>\n",
+       "      <td>11000</td>\n",
+       "      <td>0.422325</td>\n",
+       "      <td>0.332</td>\n",
+       "      <td>0.328</td>\n",
+       "      <td>0.366</td>\n",
+       "      <td>0.426</td>\n",
+       "      <td>0.188</td>\n",
+       "      <td>0.320</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.398</td>\n",
+       "      <td>0.397</td>\n",
+       "      <td>0.535</td>\n",
+       "      <td>0.529</td>\n",
+       "      <td>0.801</td>\n",
+       "      <td>0.695</td>\n",
+       "      <td>0.3775</td>\n",
+       "      <td>0.3800</td>\n",
+       "      <td>0.267457</td>\n",
+       "      <td>0.285596</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>12</th>\n",
+       "      <td>deduped_removed_cross</td>\n",
+       "      <td>5</td>\n",
+       "      <td>12000</td>\n",
+       "      <td>0.420167</td>\n",
+       "      <td>0.348</td>\n",
+       "      <td>0.324</td>\n",
+       "      <td>0.364</td>\n",
+       "      <td>0.434</td>\n",
+       "      <td>0.194</td>\n",
+       "      <td>0.306</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.377</td>\n",
+       "      <td>0.392</td>\n",
+       "      <td>0.541</td>\n",
+       "      <td>0.527</td>\n",
+       "      <td>0.790</td>\n",
+       "      <td>0.690</td>\n",
+       "      <td>0.3680</td>\n",
+       "      <td>0.3755</td>\n",
+       "      <td>0.267547</td>\n",
+       "      <td>0.285836</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13</th>\n",
+       "      <td>deduped_removed_cross</td>\n",
+       "      <td>5</td>\n",
+       "      <td>13000</td>\n",
+       "      <td>0.422913</td>\n",
+       "      <td>0.346</td>\n",
+       "      <td>0.330</td>\n",
+       "      <td>0.372</td>\n",
+       "      <td>0.438</td>\n",
+       "      <td>0.190</td>\n",
+       "      <td>0.320</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.392</td>\n",
+       "      <td>0.396</td>\n",
+       "      <td>0.540</td>\n",
+       "      <td>0.522</td>\n",
+       "      <td>0.802</td>\n",
+       "      <td>0.707</td>\n",
+       "      <td>0.3760</td>\n",
+       "      <td>0.3845</td>\n",
+       "      <td>0.271108</td>\n",
+       "      <td>0.287802</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>14</th>\n",
+       "      <td>deduped_removed_cross</td>\n",
+       "      <td>5</td>\n",
+       "      <td>13500</td>\n",
+       "      <td>0.421868</td>\n",
+       "      <td>0.345</td>\n",
+       "      <td>0.322</td>\n",
+       "      <td>0.370</td>\n",
+       "      <td>0.431</td>\n",
+       "      <td>0.202</td>\n",
+       "      <td>0.330</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.387</td>\n",
+       "      <td>0.392</td>\n",
+       "      <td>0.540</td>\n",
+       "      <td>0.516</td>\n",
+       "      <td>0.797</td>\n",
+       "      <td>0.700</td>\n",
+       "      <td>0.3790</td>\n",
+       "      <td>0.3870</td>\n",
+       "      <td>0.269510</td>\n",
+       "      <td>0.287944</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>15</th>\n",
+       "      <td>deduped_removed_cross</td>\n",
+       "      <td>6</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.330893</td>\n",
+       "      <td>0.186</td>\n",
+       "      <td>0.233</td>\n",
+       "      <td>0.272</td>\n",
+       "      <td>0.258</td>\n",
+       "      <td>0.166</td>\n",
+       "      <td>0.286</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.367</td>\n",
+       "      <td>0.362</td>\n",
+       "      <td>0.516</td>\n",
+       "      <td>0.497</td>\n",
+       "      <td>0.208</td>\n",
+       "      <td>0.202</td>\n",
+       "      <td>0.2195</td>\n",
+       "      <td>0.2510</td>\n",
+       "      <td>0.230294</td>\n",
+       "      <td>0.250147</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>16</th>\n",
+       "      <td>deduped_removed_cross</td>\n",
+       "      <td>6</td>\n",
+       "      <td>1000</td>\n",
+       "      <td>0.360039</td>\n",
+       "      <td>0.236</td>\n",
+       "      <td>0.259</td>\n",
+       "      <td>0.283</td>\n",
+       "      <td>0.277</td>\n",
+       "      <td>0.130</td>\n",
+       "      <td>0.274</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.354</td>\n",
+       "      <td>0.386</td>\n",
+       "      <td>0.509</td>\n",
+       "      <td>0.507</td>\n",
+       "      <td>0.559</td>\n",
+       "      <td>0.500</td>\n",
+       "      <td>0.2590</td>\n",
+       "      <td>0.2970</td>\n",
+       "      <td>0.243455</td>\n",
+       "      <td>0.254311</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>17</th>\n",
+       "      <td>deduped_removed_cross</td>\n",
+       "      <td>6</td>\n",
+       "      <td>2000</td>\n",
+       "      <td>0.371564</td>\n",
+       "      <td>0.270</td>\n",
+       "      <td>0.283</td>\n",
+       "      <td>0.303</td>\n",
+       "      <td>0.305</td>\n",
+       "      <td>0.132</td>\n",
+       "      <td>0.280</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.377</td>\n",
+       "      <td>0.392</td>\n",
+       "      <td>0.522</td>\n",
+       "      <td>0.504</td>\n",
+       "      <td>0.665</td>\n",
+       "      <td>0.566</td>\n",
+       "      <td>0.3040</td>\n",
+       "      <td>0.3135</td>\n",
+       "      <td>0.249051</td>\n",
+       "      <td>0.255010</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>18</th>\n",
+       "      <td>deduped_removed_cross</td>\n",
+       "      <td>6</td>\n",
+       "      <td>3000</td>\n",
+       "      <td>0.383770</td>\n",
+       "      <td>0.283</td>\n",
+       "      <td>0.286</td>\n",
+       "      <td>0.323</td>\n",
+       "      <td>0.320</td>\n",
+       "      <td>0.156</td>\n",
+       "      <td>0.296</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.375</td>\n",
+       "      <td>0.394</td>\n",
+       "      <td>0.503</td>\n",
+       "      <td>0.497</td>\n",
+       "      <td>0.721</td>\n",
+       "      <td>0.626</td>\n",
+       "      <td>0.3140</td>\n",
+       "      <td>0.3410</td>\n",
+       "      <td>0.254015</td>\n",
+       "      <td>0.266158</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>19</th>\n",
+       "      <td>deduped_removed_cross</td>\n",
+       "      <td>6</td>\n",
+       "      <td>4000</td>\n",
+       "      <td>0.391082</td>\n",
+       "      <td>0.293</td>\n",
+       "      <td>0.298</td>\n",
+       "      <td>0.339</td>\n",
+       "      <td>0.361</td>\n",
+       "      <td>0.160</td>\n",
+       "      <td>0.292</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.380</td>\n",
+       "      <td>0.399</td>\n",
+       "      <td>0.505</td>\n",
+       "      <td>0.494</td>\n",
+       "      <td>0.719</td>\n",
+       "      <td>0.615</td>\n",
+       "      <td>0.3375</td>\n",
+       "      <td>0.3375</td>\n",
+       "      <td>0.256696</td>\n",
+       "      <td>0.268152</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>20</th>\n",
+       "      <td>deduped_removed_cross</td>\n",
+       "      <td>6</td>\n",
+       "      <td>5000</td>\n",
+       "      <td>0.399130</td>\n",
+       "      <td>0.309</td>\n",
+       "      <td>0.311</td>\n",
+       "      <td>0.343</td>\n",
+       "      <td>0.376</td>\n",
+       "      <td>0.160</td>\n",
+       "      <td>0.286</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.392</td>\n",
+       "      <td>0.401</td>\n",
+       "      <td>0.525</td>\n",
+       "      <td>0.512</td>\n",
+       "      <td>0.733</td>\n",
+       "      <td>0.639</td>\n",
+       "      <td>0.3390</td>\n",
+       "      <td>0.3580</td>\n",
+       "      <td>0.257450</td>\n",
+       "      <td>0.271040</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>21</th>\n",
+       "      <td>deduped_removed_cross</td>\n",
+       "      <td>6</td>\n",
+       "      <td>6000</td>\n",
+       "      <td>0.402792</td>\n",
+       "      <td>0.326</td>\n",
+       "      <td>0.318</td>\n",
+       "      <td>0.353</td>\n",
+       "      <td>0.387</td>\n",
+       "      <td>0.176</td>\n",
+       "      <td>0.284</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.376</td>\n",
+       "      <td>0.405</td>\n",
+       "      <td>0.522</td>\n",
+       "      <td>0.514</td>\n",
+       "      <td>0.753</td>\n",
+       "      <td>0.664</td>\n",
+       "      <td>0.3450</td>\n",
+       "      <td>0.3645</td>\n",
+       "      <td>0.262549</td>\n",
+       "      <td>0.273836</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>22</th>\n",
+       "      <td>deduped_removed_cross</td>\n",
+       "      <td>6</td>\n",
+       "      <td>7000</td>\n",
+       "      <td>0.408846</td>\n",
+       "      <td>0.319</td>\n",
+       "      <td>0.319</td>\n",
+       "      <td>0.356</td>\n",
+       "      <td>0.407</td>\n",
+       "      <td>0.172</td>\n",
+       "      <td>0.300</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.386</td>\n",
+       "      <td>0.399</td>\n",
+       "      <td>0.521</td>\n",
+       "      <td>0.521</td>\n",
+       "      <td>0.764</td>\n",
+       "      <td>0.662</td>\n",
+       "      <td>0.3585</td>\n",
+       "      <td>0.3625</td>\n",
+       "      <td>0.262740</td>\n",
+       "      <td>0.276266</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>23</th>\n",
+       "      <td>deduped_removed_cross</td>\n",
+       "      <td>6</td>\n",
+       "      <td>8000</td>\n",
+       "      <td>0.411429</td>\n",
+       "      <td>0.314</td>\n",
+       "      <td>0.323</td>\n",
+       "      <td>0.361</td>\n",
+       "      <td>0.412</td>\n",
+       "      <td>0.168</td>\n",
+       "      <td>0.286</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.395</td>\n",
+       "      <td>0.404</td>\n",
+       "      <td>0.533</td>\n",
+       "      <td>0.511</td>\n",
+       "      <td>0.754</td>\n",
+       "      <td>0.646</td>\n",
+       "      <td>0.3555</td>\n",
+       "      <td>0.3690</td>\n",
+       "      <td>0.263875</td>\n",
+       "      <td>0.278433</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>24</th>\n",
+       "      <td>deduped_removed_cross</td>\n",
+       "      <td>6</td>\n",
+       "      <td>9000</td>\n",
+       "      <td>0.417279</td>\n",
+       "      <td>0.337</td>\n",
+       "      <td>0.329</td>\n",
+       "      <td>0.367</td>\n",
+       "      <td>0.421</td>\n",
+       "      <td>0.176</td>\n",
+       "      <td>0.294</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.407</td>\n",
+       "      <td>0.403</td>\n",
+       "      <td>0.532</td>\n",
+       "      <td>0.526</td>\n",
+       "      <td>0.775</td>\n",
+       "      <td>0.666</td>\n",
+       "      <td>0.3605</td>\n",
+       "      <td>0.3730</td>\n",
+       "      <td>0.265119</td>\n",
+       "      <td>0.283235</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>25</th>\n",
+       "      <td>deduped_removed_cross</td>\n",
+       "      <td>6</td>\n",
+       "      <td>10000</td>\n",
+       "      <td>0.421399</td>\n",
+       "      <td>0.339</td>\n",
+       "      <td>0.322</td>\n",
+       "      <td>0.376</td>\n",
+       "      <td>0.426</td>\n",
+       "      <td>0.174</td>\n",
+       "      <td>0.320</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.397</td>\n",
+       "      <td>0.401</td>\n",
+       "      <td>0.542</td>\n",
+       "      <td>0.532</td>\n",
+       "      <td>0.764</td>\n",
+       "      <td>0.673</td>\n",
+       "      <td>0.3675</td>\n",
+       "      <td>0.3840</td>\n",
+       "      <td>0.272474</td>\n",
+       "      <td>0.286190</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>26</th>\n",
+       "      <td>deduped_removed_cross</td>\n",
+       "      <td>6</td>\n",
+       "      <td>11000</td>\n",
+       "      <td>0.421204</td>\n",
+       "      <td>0.349</td>\n",
+       "      <td>0.337</td>\n",
+       "      <td>0.378</td>\n",
+       "      <td>0.428</td>\n",
+       "      <td>0.188</td>\n",
+       "      <td>0.314</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.403</td>\n",
+       "      <td>0.398</td>\n",
+       "      <td>0.530</td>\n",
+       "      <td>0.516</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.3690</td>\n",
+       "      <td>0.3780</td>\n",
+       "      <td>0.269131</td>\n",
+       "      <td>0.288633</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>27</th>\n",
+       "      <td>deduped_removed_cross</td>\n",
+       "      <td>6</td>\n",
+       "      <td>12000</td>\n",
+       "      <td>0.421667</td>\n",
+       "      <td>0.342</td>\n",
+       "      <td>0.326</td>\n",
+       "      <td>0.383</td>\n",
+       "      <td>0.434</td>\n",
+       "      <td>0.174</td>\n",
+       "      <td>0.310</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.399</td>\n",
+       "      <td>0.396</td>\n",
+       "      <td>0.538</td>\n",
+       "      <td>0.525</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.3660</td>\n",
+       "      <td>0.3810</td>\n",
+       "      <td>0.270691</td>\n",
+       "      <td>0.287333</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>28</th>\n",
+       "      <td>deduped_removed_cross</td>\n",
+       "      <td>6</td>\n",
+       "      <td>13000</td>\n",
+       "      <td>0.424979</td>\n",
+       "      <td>0.349</td>\n",
+       "      <td>0.336</td>\n",
+       "      <td>0.383</td>\n",
+       "      <td>0.440</td>\n",
+       "      <td>0.178</td>\n",
+       "      <td>0.314</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.401</td>\n",
+       "      <td>0.392</td>\n",
+       "      <td>0.535</td>\n",
+       "      <td>0.526</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.3785</td>\n",
+       "      <td>0.3905</td>\n",
+       "      <td>0.268910</td>\n",
+       "      <td>0.289335</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>29</th>\n",
+       "      <td>deduped_removed_cross</td>\n",
+       "      <td>6</td>\n",
+       "      <td>13500</td>\n",
+       "      <td>0.425356</td>\n",
+       "      <td>0.347</td>\n",
+       "      <td>0.333</td>\n",
+       "      <td>0.386</td>\n",
+       "      <td>0.444</td>\n",
+       "      <td>0.186</td>\n",
+       "      <td>0.322</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.406</td>\n",
+       "      <td>0.392</td>\n",
+       "      <td>0.543</td>\n",
+       "      <td>0.527</td>\n",
+       "      <td>0.783</td>\n",
+       "      <td>0.682</td>\n",
+       "      <td>0.3745</td>\n",
+       "      <td>0.3890</td>\n",
+       "      <td>0.270869</td>\n",
+       "      <td>0.289845</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>30</th>\n",
+       "      <td>cross_minhash_dump_CC-MAIN-2013-48</td>\n",
+       "      <td>6</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.331018</td>\n",
+       "      <td>0.186</td>\n",
+       "      <td>0.233</td>\n",
+       "      <td>0.272</td>\n",
+       "      <td>0.258</td>\n",
+       "      <td>0.166</td>\n",
+       "      <td>0.286</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.367</td>\n",
+       "      <td>0.362</td>\n",
+       "      <td>0.515</td>\n",
+       "      <td>0.497</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.2195</td>\n",
+       "      <td>0.2520</td>\n",
+       "      <td>0.230228</td>\n",
+       "      <td>0.250147</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>31</th>\n",
+       "      <td>cross_minhash_dump_CC-MAIN-2013-48</td>\n",
+       "      <td>6</td>\n",
+       "      <td>1000</td>\n",
+       "      <td>0.349494</td>\n",
+       "      <td>0.217</td>\n",
+       "      <td>0.248</td>\n",
+       "      <td>0.288</td>\n",
+       "      <td>0.286</td>\n",
+       "      <td>0.104</td>\n",
+       "      <td>0.244</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.366</td>\n",
+       "      <td>0.380</td>\n",
+       "      <td>0.499</td>\n",
+       "      <td>0.492</td>\n",
+       "      <td>0.546</td>\n",
+       "      <td>0.484</td>\n",
+       "      <td>0.2565</td>\n",
+       "      <td>0.2780</td>\n",
+       "      <td>0.239651</td>\n",
+       "      <td>0.253956</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>32</th>\n",
+       "      <td>cross_minhash_dump_CC-MAIN-2013-48</td>\n",
+       "      <td>6</td>\n",
+       "      <td>2000</td>\n",
+       "      <td>0.367893</td>\n",
+       "      <td>0.245</td>\n",
+       "      <td>0.280</td>\n",
+       "      <td>0.298</td>\n",
+       "      <td>0.288</td>\n",
+       "      <td>0.128</td>\n",
+       "      <td>0.280</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.366</td>\n",
+       "      <td>0.383</td>\n",
+       "      <td>0.519</td>\n",
+       "      <td>0.499</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.2845</td>\n",
+       "      <td>0.3115</td>\n",
+       "      <td>0.239715</td>\n",
+       "      <td>0.253644</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>33</th>\n",
+       "      <td>cross_minhash_dump_CC-MAIN-2013-48</td>\n",
+       "      <td>6</td>\n",
+       "      <td>3000</td>\n",
+       "      <td>0.379114</td>\n",
+       "      <td>0.269</td>\n",
+       "      <td>0.291</td>\n",
+       "      <td>0.304</td>\n",
+       "      <td>0.328</td>\n",
+       "      <td>0.138</td>\n",
+       "      <td>0.266</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.362</td>\n",
+       "      <td>0.394</td>\n",
+       "      <td>0.519</td>\n",
+       "      <td>0.504</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.3035</td>\n",
+       "      <td>0.3335</td>\n",
+       "      <td>0.250551</td>\n",
+       "      <td>0.262409</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>34</th>\n",
+       "      <td>cross_minhash_dump_CC-MAIN-2013-48</td>\n",
+       "      <td>6</td>\n",
+       "      <td>4000</td>\n",
+       "      <td>0.383025</td>\n",
+       "      <td>0.277</td>\n",
+       "      <td>0.289</td>\n",
+       "      <td>0.311</td>\n",
+       "      <td>0.338</td>\n",
+       "      <td>0.132</td>\n",
+       "      <td>0.280</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.361</td>\n",
+       "      <td>0.393</td>\n",
+       "      <td>0.502</td>\n",
+       "      <td>0.496</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.3105</td>\n",
+       "      <td>0.3375</td>\n",
+       "      <td>0.249887</td>\n",
+       "      <td>0.263702</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>35</th>\n",
+       "      <td>cross_minhash_dump_CC-MAIN-2013-48</td>\n",
+       "      <td>6</td>\n",
+       "      <td>5000</td>\n",
+       "      <td>0.387223</td>\n",
+       "      <td>0.290</td>\n",
+       "      <td>0.306</td>\n",
+       "      <td>0.327</td>\n",
+       "      <td>0.356</td>\n",
+       "      <td>0.138</td>\n",
+       "      <td>0.276</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.365</td>\n",
+       "      <td>0.389</td>\n",
+       "      <td>0.515</td>\n",
+       "      <td>0.511</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.3190</td>\n",
+       "      <td>0.3380</td>\n",
+       "      <td>0.252621</td>\n",
+       "      <td>0.266785</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>36</th>\n",
+       "      <td>cross_minhash_dump_CC-MAIN-2013-48</td>\n",
+       "      <td>6</td>\n",
+       "      <td>6000</td>\n",
+       "      <td>0.394011</td>\n",
+       "      <td>0.303</td>\n",
+       "      <td>0.305</td>\n",
+       "      <td>0.332</td>\n",
+       "      <td>0.356</td>\n",
+       "      <td>0.142</td>\n",
+       "      <td>0.288</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.375</td>\n",
+       "      <td>0.397</td>\n",
+       "      <td>0.540</td>\n",
+       "      <td>0.521</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.3280</td>\n",
+       "      <td>0.3515</td>\n",
+       "      <td>0.252255</td>\n",
+       "      <td>0.265589</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>37</th>\n",
+       "      <td>cross_minhash_dump_CC-MAIN-2013-48</td>\n",
+       "      <td>6</td>\n",
+       "      <td>7000</td>\n",
+       "      <td>0.398090</td>\n",
+       "      <td>0.316</td>\n",
+       "      <td>0.305</td>\n",
+       "      <td>0.337</td>\n",
+       "      <td>0.359</td>\n",
+       "      <td>0.142</td>\n",
+       "      <td>0.302</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.372</td>\n",
+       "      <td>0.401</td>\n",
+       "      <td>0.531</td>\n",
+       "      <td>0.510</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.3320</td>\n",
+       "      <td>0.3550</td>\n",
+       "      <td>0.250146</td>\n",
+       "      <td>0.267719</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>38</th>\n",
+       "      <td>cross_minhash_dump_CC-MAIN-2013-48</td>\n",
+       "      <td>6</td>\n",
+       "      <td>8000</td>\n",
+       "      <td>0.398513</td>\n",
+       "      <td>0.326</td>\n",
+       "      <td>0.315</td>\n",
+       "      <td>0.339</td>\n",
+       "      <td>0.372</td>\n",
+       "      <td>0.150</td>\n",
+       "      <td>0.288</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.372</td>\n",
+       "      <td>0.396</td>\n",
+       "      <td>0.532</td>\n",
+       "      <td>0.508</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.3365</td>\n",
+       "      <td>0.3630</td>\n",
+       "      <td>0.258433</td>\n",
+       "      <td>0.274100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>39</th>\n",
+       "      <td>cross_minhash_dump_CC-MAIN-2013-48</td>\n",
+       "      <td>6</td>\n",
+       "      <td>9000</td>\n",
+       "      <td>0.397494</td>\n",
+       "      <td>0.310</td>\n",
+       "      <td>0.314</td>\n",
+       "      <td>0.345</td>\n",
+       "      <td>0.374</td>\n",
+       "      <td>0.140</td>\n",
+       "      <td>0.274</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.364</td>\n",
+       "      <td>0.392</td>\n",
+       "      <td>0.529</td>\n",
+       "      <td>0.506</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.3445</td>\n",
+       "      <td>0.3610</td>\n",
+       "      <td>0.258927</td>\n",
+       "      <td>0.271955</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>40</th>\n",
+       "      <td>cross_minhash_dump_CC-MAIN-2013-48</td>\n",
+       "      <td>6</td>\n",
+       "      <td>10000</td>\n",
+       "      <td>0.402640</td>\n",
+       "      <td>0.321</td>\n",
+       "      <td>0.327</td>\n",
+       "      <td>0.347</td>\n",
+       "      <td>0.383</td>\n",
+       "      <td>0.156</td>\n",
+       "      <td>0.280</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.376</td>\n",
+       "      <td>0.397</td>\n",
+       "      <td>0.529</td>\n",
+       "      <td>0.513</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.3445</td>\n",
+       "      <td>0.3650</td>\n",
+       "      <td>0.258294</td>\n",
+       "      <td>0.272123</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>41</th>\n",
+       "      <td>cross_minhash_dump_CC-MAIN-2013-48</td>\n",
+       "      <td>6</td>\n",
+       "      <td>11000</td>\n",
+       "      <td>0.402599</td>\n",
+       "      <td>0.318</td>\n",
+       "      <td>0.322</td>\n",
+       "      <td>0.348</td>\n",
+       "      <td>0.381</td>\n",
+       "      <td>0.160</td>\n",
+       "      <td>0.284</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.367</td>\n",
+       "      <td>0.387</td>\n",
+       "      <td>0.538</td>\n",
+       "      <td>0.516</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.3490</td>\n",
+       "      <td>0.3660</td>\n",
+       "      <td>0.259610</td>\n",
+       "      <td>0.276792</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>42</th>\n",
+       "      <td>cross_minhash_dump_CC-MAIN-2013-48</td>\n",
+       "      <td>6</td>\n",
+       "      <td>12000</td>\n",
+       "      <td>0.407442</td>\n",
+       "      <td>0.328</td>\n",
+       "      <td>0.319</td>\n",
+       "      <td>0.349</td>\n",
+       "      <td>0.395</td>\n",
+       "      <td>0.162</td>\n",
+       "      <td>0.290</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.367</td>\n",
+       "      <td>0.407</td>\n",
+       "      <td>0.528</td>\n",
+       "      <td>0.510</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.3510</td>\n",
+       "      <td>0.3700</td>\n",
+       "      <td>0.260350</td>\n",
+       "      <td>0.279535</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>43</th>\n",
+       "      <td>cross_minhash_dump_CC-MAIN-2013-48</td>\n",
+       "      <td>6</td>\n",
+       "      <td>13000</td>\n",
+       "      <td>0.405577</td>\n",
+       "      <td>0.324</td>\n",
+       "      <td>0.318</td>\n",
+       "      <td>0.350</td>\n",
+       "      <td>0.385</td>\n",
+       "      <td>0.158</td>\n",
+       "      <td>0.290</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.373</td>\n",
+       "      <td>0.396</td>\n",
+       "      <td>0.538</td>\n",
+       "      <td>0.510</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.3540</td>\n",
+       "      <td>0.3730</td>\n",
+       "      <td>0.258481</td>\n",
+       "      <td>0.274616</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>44</th>\n",
+       "      <td>cross_minhash_dump_CC-MAIN-2013-48</td>\n",
+       "      <td>6</td>\n",
+       "      <td>13500</td>\n",
+       "      <td>0.405000</td>\n",
+       "      <td>0.320</td>\n",
+       "      <td>0.312</td>\n",
+       "      <td>0.354</td>\n",
+       "      <td>0.393</td>\n",
+       "      <td>0.152</td>\n",
+       "      <td>0.288</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.367</td>\n",
+       "      <td>0.396</td>\n",
+       "      <td>0.528</td>\n",
+       "      <td>0.513</td>\n",
+       "      <td>0.785</td>\n",
+       "      <td>0.675</td>\n",
+       "      <td>0.3590</td>\n",
+       "      <td>0.3660</td>\n",
+       "      <td>0.260174</td>\n",
+       "      <td>0.278002</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>45 rows × 22 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                               runname  seed  steps  agg_score  \\\n",
+       "0                deduped_removed_cross     5      0   0.330893   \n",
+       "1                deduped_removed_cross     5   1000   0.354090   \n",
+       "2                deduped_removed_cross     5   2000   0.373601   \n",
+       "3                deduped_removed_cross     5   3000   0.383122   \n",
+       "4                deduped_removed_cross     5   4000   0.390222   \n",
+       "5                deduped_removed_cross     5   5000   0.400239   \n",
+       "6                deduped_removed_cross     5   6000   0.401484   \n",
+       "7                deduped_removed_cross     5   7000   0.403533   \n",
+       "8                deduped_removed_cross     5   8000   0.411774   \n",
+       "9                deduped_removed_cross     5   9000   0.410993   \n",
+       "10               deduped_removed_cross     5  10000   0.417883   \n",
+       "11               deduped_removed_cross     5  11000   0.422325   \n",
+       "12               deduped_removed_cross     5  12000   0.420167   \n",
+       "13               deduped_removed_cross     5  13000   0.422913   \n",
+       "14               deduped_removed_cross     5  13500   0.421868   \n",
+       "15               deduped_removed_cross     6      0   0.330893   \n",
+       "16               deduped_removed_cross     6   1000   0.360039   \n",
+       "17               deduped_removed_cross     6   2000   0.371564   \n",
+       "18               deduped_removed_cross     6   3000   0.383770   \n",
+       "19               deduped_removed_cross     6   4000   0.391082   \n",
+       "20               deduped_removed_cross     6   5000   0.399130   \n",
+       "21               deduped_removed_cross     6   6000   0.402792   \n",
+       "22               deduped_removed_cross     6   7000   0.408846   \n",
+       "23               deduped_removed_cross     6   8000   0.411429   \n",
+       "24               deduped_removed_cross     6   9000   0.417279   \n",
+       "25               deduped_removed_cross     6  10000   0.421399   \n",
+       "26               deduped_removed_cross     6  11000   0.421204   \n",
+       "27               deduped_removed_cross     6  12000   0.421667   \n",
+       "28               deduped_removed_cross     6  13000   0.424979   \n",
+       "29               deduped_removed_cross     6  13500   0.425356   \n",
+       "30  cross_minhash_dump_CC-MAIN-2013-48     6      0   0.331018   \n",
+       "31  cross_minhash_dump_CC-MAIN-2013-48     6   1000   0.349494   \n",
+       "32  cross_minhash_dump_CC-MAIN-2013-48     6   2000   0.367893   \n",
+       "33  cross_minhash_dump_CC-MAIN-2013-48     6   3000   0.379114   \n",
+       "34  cross_minhash_dump_CC-MAIN-2013-48     6   4000   0.383025   \n",
+       "35  cross_minhash_dump_CC-MAIN-2013-48     6   5000   0.387223   \n",
+       "36  cross_minhash_dump_CC-MAIN-2013-48     6   6000   0.394011   \n",
+       "37  cross_minhash_dump_CC-MAIN-2013-48     6   7000   0.398090   \n",
+       "38  cross_minhash_dump_CC-MAIN-2013-48     6   8000   0.398513   \n",
+       "39  cross_minhash_dump_CC-MAIN-2013-48     6   9000   0.397494   \n",
+       "40  cross_minhash_dump_CC-MAIN-2013-48     6  10000   0.402640   \n",
+       "41  cross_minhash_dump_CC-MAIN-2013-48     6  11000   0.402599   \n",
+       "42  cross_minhash_dump_CC-MAIN-2013-48     6  12000   0.407442   \n",
+       "43  cross_minhash_dump_CC-MAIN-2013-48     6  13000   0.405577   \n",
+       "44  cross_minhash_dump_CC-MAIN-2013-48     6  13500   0.405000   \n",
+       "\n",
+       "    commonsense_qa/acc  commonsense_qa/acc_norm  hellaswag/acc  \\\n",
+       "0                0.186                    0.233          0.272   \n",
+       "1                0.253                    0.257          0.290   \n",
+       "2                0.274                    0.290          0.313   \n",
+       "3                0.306                    0.292          0.323   \n",
+       "4                0.300                    0.292          0.324   \n",
+       "5                0.322                    0.308          0.325   \n",
+       "6                0.315                    0.314          0.341   \n",
+       "7                0.324                    0.315          0.350   \n",
+       "8                0.344                    0.313          0.352   \n",
+       "9                0.335                    0.322          0.361   \n",
+       "10               0.330                    0.320          0.370   \n",
+       "11               0.332                    0.328          0.366   \n",
+       "12               0.348                    0.324          0.364   \n",
+       "13               0.346                    0.330          0.372   \n",
+       "14               0.345                    0.322          0.370   \n",
+       "15               0.186                    0.233          0.272   \n",
+       "16               0.236                    0.259          0.283   \n",
+       "17               0.270                    0.283          0.303   \n",
+       "18               0.283                    0.286          0.323   \n",
+       "19               0.293                    0.298          0.339   \n",
+       "20               0.309                    0.311          0.343   \n",
+       "21               0.326                    0.318          0.353   \n",
+       "22               0.319                    0.319          0.356   \n",
+       "23               0.314                    0.323          0.361   \n",
+       "24               0.337                    0.329          0.367   \n",
+       "25               0.339                    0.322          0.376   \n",
+       "26               0.349                    0.337          0.378   \n",
+       "27               0.342                    0.326          0.383   \n",
+       "28               0.349                    0.336          0.383   \n",
+       "29               0.347                    0.333          0.386   \n",
+       "30               0.186                    0.233          0.272   \n",
+       "31               0.217                    0.248          0.288   \n",
+       "32               0.245                    0.280          0.298   \n",
+       "33               0.269                    0.291          0.304   \n",
+       "34               0.277                    0.289          0.311   \n",
+       "35               0.290                    0.306          0.327   \n",
+       "36               0.303                    0.305          0.332   \n",
+       "37               0.316                    0.305          0.337   \n",
+       "38               0.326                    0.315          0.339   \n",
+       "39               0.310                    0.314          0.345   \n",
+       "40               0.321                    0.327          0.347   \n",
+       "41               0.318                    0.322          0.348   \n",
+       "42               0.328                    0.319          0.349   \n",
+       "43               0.324                    0.318          0.350   \n",
+       "44               0.320                    0.312          0.354   \n",
+       "\n",
+       "    hellaswag/acc_norm  openbookqa/acc  openbookqa/acc_norm  ...  siqa/acc  \\\n",
+       "0                0.258           0.166                0.286  ...     0.367   \n",
+       "1                0.278           0.124                0.264  ...     0.368   \n",
+       "2                0.312           0.116                0.258  ...     0.367   \n",
+       "3                0.335           0.150                0.278  ...     0.371   \n",
+       "4                0.351           0.144                0.278  ...     0.386   \n",
+       "5                0.364           0.172                0.298  ...     0.382   \n",
+       "6                0.372           0.162                0.314  ...     0.377   \n",
+       "7                0.386           0.188                0.298  ...     0.376   \n",
+       "8                0.409           0.170                0.310  ...     0.374   \n",
+       "9                0.404           0.182                0.294  ...     0.374   \n",
+       "10               0.417           0.192                0.324  ...     0.389   \n",
+       "11               0.426           0.188                0.320  ...     0.398   \n",
+       "12               0.434           0.194                0.306  ...     0.377   \n",
+       "13               0.438           0.190                0.320  ...     0.392   \n",
+       "14               0.431           0.202                0.330  ...     0.387   \n",
+       "15               0.258           0.166                0.286  ...     0.367   \n",
+       "16               0.277           0.130                0.274  ...     0.354   \n",
+       "17               0.305           0.132                0.280  ...     0.377   \n",
+       "18               0.320           0.156                0.296  ...     0.375   \n",
+       "19               0.361           0.160                0.292  ...     0.380   \n",
+       "20               0.376           0.160                0.286  ...     0.392   \n",
+       "21               0.387           0.176                0.284  ...     0.376   \n",
+       "22               0.407           0.172                0.300  ...     0.386   \n",
+       "23               0.412           0.168                0.286  ...     0.395   \n",
+       "24               0.421           0.176                0.294  ...     0.407   \n",
+       "25               0.426           0.174                0.320  ...     0.397   \n",
+       "26               0.428           0.188                0.314  ...     0.403   \n",
+       "27               0.434           0.174                0.310  ...     0.399   \n",
+       "28               0.440           0.178                0.314  ...     0.401   \n",
+       "29               0.444           0.186                0.322  ...     0.406   \n",
+       "30               0.258           0.166                0.286  ...     0.367   \n",
+       "31               0.286           0.104                0.244  ...     0.366   \n",
+       "32               0.288           0.128                0.280  ...     0.366   \n",
+       "33               0.328           0.138                0.266  ...     0.362   \n",
+       "34               0.338           0.132                0.280  ...     0.361   \n",
+       "35               0.356           0.138                0.276  ...     0.365   \n",
+       "36               0.356           0.142                0.288  ...     0.375   \n",
+       "37               0.359           0.142                0.302  ...     0.372   \n",
+       "38               0.372           0.150                0.288  ...     0.372   \n",
+       "39               0.374           0.140                0.274  ...     0.364   \n",
+       "40               0.383           0.156                0.280  ...     0.376   \n",
+       "41               0.381           0.160                0.284  ...     0.367   \n",
+       "42               0.395           0.162                0.290  ...     0.367   \n",
+       "43               0.385           0.158                0.290  ...     0.373   \n",
+       "44               0.393           0.152                0.288  ...     0.367   \n",
+       "\n",
+       "    siqa/acc_norm  winogrande/acc  winogrande/acc_norm  sciq/acc  \\\n",
+       "0           0.362           0.516                0.497     0.208   \n",
+       "1           0.389           0.509                0.491     0.582   \n",
+       "2           0.397           0.516                0.505     0.686   \n",
+       "3           0.401           0.513                0.500     0.712   \n",
+       "4           0.395           0.511                0.511     0.750   \n",
+       "5           0.398           0.518                0.522     0.751   \n",
+       "6           0.390           0.498                0.492     0.776   \n",
+       "7           0.384           0.518                0.521     0.769   \n",
+       "8           0.391           0.530                0.521     0.781   \n",
+       "9           0.391           0.526                0.514     0.769   \n",
+       "10          0.389           0.518                0.524     0.785   \n",
+       "11          0.397           0.535                0.529     0.801   \n",
+       "12          0.392           0.541                0.527     0.790   \n",
+       "13          0.396           0.540                0.522     0.802   \n",
+       "14          0.392           0.540                0.516     0.797   \n",
+       "15          0.362           0.516                0.497     0.208   \n",
+       "16          0.386           0.509                0.507     0.559   \n",
+       "17          0.392           0.522                0.504     0.665   \n",
+       "18          0.394           0.503                0.497     0.721   \n",
+       "19          0.399           0.505                0.494     0.719   \n",
+       "20          0.401           0.525                0.512     0.733   \n",
+       "21          0.405           0.522                0.514     0.753   \n",
+       "22          0.399           0.521                0.521     0.764   \n",
+       "23          0.404           0.533                0.511     0.754   \n",
+       "24          0.403           0.532                0.526     0.775   \n",
+       "25          0.401           0.542                0.532     0.764   \n",
+       "26          0.398           0.530                0.516       NaN   \n",
+       "27          0.396           0.538                0.525       NaN   \n",
+       "28          0.392           0.535                0.526       NaN   \n",
+       "29          0.392           0.543                0.527     0.783   \n",
+       "30          0.362           0.515                0.497       NaN   \n",
+       "31          0.380           0.499                0.492     0.546   \n",
+       "32          0.383           0.519                0.499       NaN   \n",
+       "33          0.394           0.519                0.504       NaN   \n",
+       "34          0.393           0.502                0.496       NaN   \n",
+       "35          0.389           0.515                0.511       NaN   \n",
+       "36          0.397           0.540                0.521       NaN   \n",
+       "37          0.401           0.531                0.510       NaN   \n",
+       "38          0.396           0.532                0.508       NaN   \n",
+       "39          0.392           0.529                0.506       NaN   \n",
+       "40          0.397           0.529                0.513       NaN   \n",
+       "41          0.387           0.538                0.516       NaN   \n",
+       "42          0.407           0.528                0.510       NaN   \n",
+       "43          0.396           0.538                0.510       NaN   \n",
+       "44          0.396           0.528                0.513     0.785   \n",
+       "\n",
+       "    sciq/acc_norm  arc/acc  arc/acc_norm  mmlu/acc  mmlu/acc_norm  \n",
+       "0           0.202   0.2195        0.2510  0.230294       0.250147  \n",
+       "1           0.516   0.2825        0.2955  0.239520       0.253223  \n",
+       "2           0.582   0.3090        0.3200  0.247320       0.262812  \n",
+       "3           0.611   0.3075        0.3415  0.248568       0.263474  \n",
+       "4           0.658   0.3260        0.3445  0.259246       0.273276  \n",
+       "5           0.661   0.3470        0.3545  0.258485       0.271414  \n",
+       "6           0.669   0.3530        0.3565  0.261842       0.276371  \n",
+       "7           0.672   0.3625        0.3585  0.265558       0.274768  \n",
+       "8           0.677   0.3530        0.3615  0.267141       0.283691  \n",
+       "9           0.672   0.3630        0.3715  0.266464       0.284446  \n",
+       "10          0.682   0.3735        0.3745  0.268085       0.283562  \n",
+       "11          0.695   0.3775        0.3800  0.267457       0.285596  \n",
+       "12          0.690   0.3680        0.3755  0.267547       0.285836  \n",
+       "13          0.707   0.3760        0.3845  0.271108       0.287802  \n",
+       "14          0.700   0.3790        0.3870  0.269510       0.287944  \n",
+       "15          0.202   0.2195        0.2510  0.230294       0.250147  \n",
+       "16          0.500   0.2590        0.2970  0.243455       0.254311  \n",
+       "17          0.566   0.3040        0.3135  0.249051       0.255010  \n",
+       "18          0.626   0.3140        0.3410  0.254015       0.266158  \n",
+       "19          0.615   0.3375        0.3375  0.256696       0.268152  \n",
+       "20          0.639   0.3390        0.3580  0.257450       0.271040  \n",
+       "21          0.664   0.3450        0.3645  0.262549       0.273836  \n",
+       "22          0.662   0.3585        0.3625  0.262740       0.276266  \n",
+       "23          0.646   0.3555        0.3690  0.263875       0.278433  \n",
+       "24          0.666   0.3605        0.3730  0.265119       0.283235  \n",
+       "25          0.673   0.3675        0.3840  0.272474       0.286190  \n",
+       "26            NaN   0.3690        0.3780  0.269131       0.288633  \n",
+       "27            NaN   0.3660        0.3810  0.270691       0.287333  \n",
+       "28            NaN   0.3785        0.3905  0.268910       0.289335  \n",
+       "29          0.682   0.3745        0.3890  0.270869       0.289845  \n",
+       "30            NaN   0.2195        0.2520  0.230228       0.250147  \n",
+       "31          0.484   0.2565        0.2780  0.239651       0.253956  \n",
+       "32            NaN   0.2845        0.3115  0.239715       0.253644  \n",
+       "33            NaN   0.3035        0.3335  0.250551       0.262409  \n",
+       "34            NaN   0.3105        0.3375  0.249887       0.263702  \n",
+       "35            NaN   0.3190        0.3380  0.252621       0.266785  \n",
+       "36            NaN   0.3280        0.3515  0.252255       0.265589  \n",
+       "37            NaN   0.3320        0.3550  0.250146       0.267719  \n",
+       "38            NaN   0.3365        0.3630  0.258433       0.274100  \n",
+       "39            NaN   0.3445        0.3610  0.258927       0.271955  \n",
+       "40            NaN   0.3445        0.3650  0.258294       0.272123  \n",
+       "41            NaN   0.3490        0.3660  0.259610       0.276792  \n",
+       "42            NaN   0.3510        0.3700  0.260350       0.279535  \n",
+       "43            NaN   0.3540        0.3730  0.258481       0.274616  \n",
+       "44          0.675   0.3590        0.3660  0.260174       0.278002  \n",
+       "\n",
+       "[45 rows x 22 columns]"
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "from matplotlib.figure import Figure\n",
+    "\n",
+    "df = pd.read_csv(\"../src_data/removed_data_cross_dedup.csv\")\n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "b610f43caefdf01",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-04-30T13:29:05.776714Z",
+     "start_time": "2024-04-30T13:29:05.774546Z"
+    },
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "runs_mapping = {\n",
+    "    \"deduped_removed_cross\": \"Originally removed data\",\n",
+    "    \"cross_minhash_dump_CC-MAIN-2013-48\": \"Originally kept data\",\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "id": "18b2dde6",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Index(['runname', 'seed', 'steps', 'agg_score', 'commonsense_qa/acc',\n",
+       "       'commonsense_qa/acc_norm', 'hellaswag/acc', 'hellaswag/acc_norm',\n",
+       "       'openbookqa/acc', 'openbookqa/acc_norm', 'piqa/acc', 'piqa/acc_norm',\n",
+       "       'siqa/acc', 'siqa/acc_norm', 'winogrande/acc', 'winogrande/acc_norm',\n",
+       "       'sciq/acc', 'sciq/acc_norm', 'arc/acc', 'arc/acc_norm', 'mmlu/acc',\n",
+       "       'mmlu/acc_norm'],\n",
+       "      dtype='object')"
+      ]
+     },
+     "execution_count": 25,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.columns"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "id": "initial_id",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-04-30T13:31:10.740797Z",
+     "start_time": "2024-04-30T13:31:10.661359Z"
+    },
+    "collapsed": true
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
+     ]
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkIAAAHHCAYAAABTMjf2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAA9hAAAPYQGoP6dpAABMi0lEQVR4nO3de3zP9f//8ft7Y+9tZhvGNowxxzn2mfg4JUyLUio5lSF0ksRHReSQYyX0ccihQj4VRfoocmhRkZJjyqGcfbAh2Zjz9vz90W+vr7dt7M0O5nW7Xi67XLyf7+fr9Xq8Xq/36/2+e72e79fbYYwxAgAAsCGPvC4AAAAgrxCEAACAbRGEAACAbRGEAACAbRGEAACAbRGEAACAbRGEAACAbRGEAACAbRGEAACAbRGEctHq1avlcDi0YMGCvC7luvbv3y+Hw6HZs2ff0PQOh0PDhg3L1pqudvfdd+vuu++2Ht9szbi+tNfw6tWrb2j6YcOGyeFwZG9RyFRCQoLatm2rYsWKyeFwaOLEiXldUo4JDw/X/ffff9ssJyfc7PF7uyII3SSHw5GlP154wM2ZOnVqvgi527dv17Bhw7R///68LkV9+/bV8uXLNXDgQM2dO1f33ntvXpd0U26lbZvbRo8erc8//zzPlr906dIc/89tXimQ1wXkd3PnznV5/MEHH2jlypXp2qtWraodO3bkZmk3pWzZsjp37pwKFix4Q9OfO3dOBQrw8kL2mTp1qoKCgtS1a9e8LuWatm/fruHDh+vuu+9WeHh4ntbyzTff6MEHH1T//v3ztI7scitt29w2evRotW3bVm3atMmT5S9dulRTpky5LcMQn1Q36fHHH3d5/OOPP2rlypXp2iXliyB0+fJlpaamysvLS97e3jc8n5uZ9laWnJysQoUK5XUZQKauPIaPHTumwMDAbJv3+fPn5eXlJQ8PLibg9sGrOQ+kpqZq1KhRKl26tLy9vdW8eXPt3r07Xb+ffvpJ9957rwICAuTr66smTZpo7dq1WVrGsWPH1L17dwUHB8vb21u1atXSnDlzXPqkjakZN26cJk6cqIiICDmdTm3fvj3T8TaffvqpIiMj5e3trerVq2vRokXq2rVruv+dXT1GKG1syO7du9W1a1cFBgYqICBA3bp109mzZ12mnTVrlpo1a6YSJUrI6XQqMjJS77zzTpbW++r5OBwObd68Od1zo0ePlqenpw4fPpzp9Gk1b9++XZ06dVKRIkXUqFEj6/n//Oc/ioqKko+Pj4oWLaoOHTro0KFDLvO4++67Vb16df3yyy9q0qSJfH19VaFCBWuc2Lfffqt69erJx8dHlStX1tdff52ujs2bN6tly5by9/eXn5+fmjdvrh9//NF6fsOGDXI4HOn2ryQtX75cDodDX375pdV2+PBhPfHEEwoODpbT6VS1atX0/vvvp5v2f//7n9q0aaNChQqpRIkS6tu3ry5cuJDp9rramjVrdOedd8rb21sRERGaPn16hv2ysr/Dw8P122+/6dtvv7UuN6eNDzt58qT69++vGjVqyM/PT/7+/mrZsqW2bt2apTovX76sESNGWK//8PBwvfLKK+nWNbNxb+Hh4dZZqtmzZ+vRRx+VJDVt2jTdpfG08SUrVqxQ7dq15e3trcjISH322Wfp5nvq1Cm98MILCgsLk9PpVIUKFfT6668rNTXV6pPZMTx16lQ5HA4ZYzRlyhSrjjR79+7Vo48+qqJFi8rX11f//Oc/tWTJEpflp40nmTdvngYPHqxSpUrJ19dXSUlJ6tq1q/z8/HTw4EHdf//98vPzU6lSpTRlyhRJ0rZt29SsWTMVKlRIZcuW1UcffeQy70uXLmn48OGqWLGivL29VaxYMTVq1EgrV67MdD9db9umWbNmjerWrStvb2+VL19eH3zwwQ1tW3fMmTNHBQoU0Isvvmi1ZeX9O+09ZufOnWrXrp38/f1VrFgx9enTR+fPn7f6ORwOJScna86cOdZ6X+/MaFaP3++//16PPvqoypQpI6fTqbCwMPXt21fnzp2z+nTt2tXat1cO+Ugzbtw4NWjQQMWKFZOPj4+ioqLyxVhYi0G26tWrl8lss65atcpIMnfccYeJiooyEyZMMMOGDTO+vr6mbt26Ln3j4uKMl5eXqV+/vnnrrbfMhAkTTM2aNY2Xl5f56aefrlnD2bNnTdWqVU3BggVN3759zb///W/TuHFjI8lMnDjR6rdv3z4jyURGRpry5cubsWPHmgkTJpgDBw5Yz82aNcvq/+WXXxqHw2Fq1qxpxo8fb1599VVTpEgRU716dVO2bFmXGiSZoUOHWo+HDh1qrfvDDz9spk6danr06GEkmZdeesll2jvvvNN07drVTJgwwUyaNMncc889RpKZPHmyS78mTZqYJk2apFuftJqTkpKMj4+P+de//pVuG0VGRppmzZpdczum1RwZGWkefPBBM3XqVDNlyhRjjDEjR440DofDtG/f3kydOtUMHz7cBAUFmfDwcPPXX3+51FiyZEkTFhZmXnzxRTNp0iQTGRlpPD09zbx580xISIgZNmyYmThxoilVqpQJCAgwSUlJ1vS//vqrKVSokAkNDTUjRowwY8eONeXKlTNOp9P8+OOPVr/y5cubVq1apVuHbt26mSJFipiLFy8aY4yJj483pUuXNmFhYea1114z77zzjnnggQeMJDNhwgRrurNnz5pKlSoZb29v89JLL5mJEyeaqKgoU7NmTSPJrFq16prb7pdffjE+Pj6mTJkyZsyYMWbEiBEmODjYmv5KWdnfixYtMqVLlzZVqlQxc+fONXPnzjUrVqwwxhjz888/m4iICDNgwAAzffp089prr1nb8vDhw9es0xhjunTpYiSZtm3bmilTppjY2FgjybRp08al39Wv6TRly5Y1Xbp0McYYs2fPHvP8888bSeaVV16xao2Pj7f6VqpUyQQGBpoBAwaY8ePHmxo1ahgPDw9rfYwxJjk52dSsWdMUK1bMvPLKK2batGkmNjbWOBwO06dPH6tfZsfwt99+a+bOnWskmRYtWlh1GPP3ayA4ONgULlzYDBo0yIwfP97UqlXLeHh4mM8++8yad9r7VWRkpKldu7YZP368GTNmjElOTjZdunQx3t7eJjIy0jz99NNmypQppkGDBtbxV7JkSev1Xq1aNePp6Wn27t1rzfuVV14xDofD9OzZ08ycOdO89dZbpmPHjmbs2LGZ7qesbNvKlSub4OBg88orr5jJkyebf/zjH8bhcJhff/3V7W2bmbJly5r77rvPejx9+nTjcDjMoEGDrLasvn+nvcfUqFHDtG7d2kyePNk8/vjjRpLp3Lmz1W/u3LnG6XSaxo0bW+v9ww8/ZFqjO8dv7969TatWrczo0aPN9OnTTffu3Y2np6dp27at1eeHH34wLVq0MJKs5ae9nowxpnTp0ubZZ581kydPNuPHjzd169Y1ksyXX3553e15KyAIZbOsBKGqVauaCxcuWO1vv/22kWS2bdtmjDEmNTXVVKxY0cTExJjU1FSr39mzZ025cuVMixYtrlnDxIkTjSTzn//8x2q7ePGiqV+/vvHz87M+aNPeRP39/c2xY8dc5pFREKpRo4YpXbq0OX36tNW2evVqIynLQeiJJ55w6ffQQw+ZYsWKubSdPXs23TrFxMSY8uXLu7RdLwgZY0zHjh1NyZIlTUpKitW2adOmdP0yklZzx44dXdr3799vPD09zahRo1zat23bZgoUKODS3qRJEyPJfPTRR1bbzp07jSTj4eHhEmaWL1+erq42bdoYLy8vs2fPHqvtyJEjpnDhwuauu+6y2gYOHGgKFixoTp48abVduHDBBAYGumzz7t27m9DQUHPixAmX2jt06GACAgKsbZ/2Gvrkk0+sPsnJyaZChQpZCkJt2rQx3t7e5sCBA1bb9u3bjaenZ7rjI6v7u1q1ai77O8358+dd9q8xf78WnE6nee21165Z55YtW4wk06NHD5f2/v37G0nmm2++sdqyEoSMMebTTz/NdBuVLVvWSDILFy602hITE01oaKi54447rLYRI0aYQoUKmd9//91l+gEDBhhPT09z8OBBaz0zO4bTau7Vq5dL2wsvvGAkme+//95qO336tClXrpwJDw+3tmXa+1X58uXT7aO08Dh69Gir7a+//jI+Pj7G4XCYefPmWe1pr/crt12tWrVcwkRWZWXbfvfdd1bbsWPHjNPpdPnPUFa3bWauDEJvv/22cTgcZsSIEdbz7rx/p73HPPDAAy7LePbZZ40ks3XrVqutUKFCLq+za3Hn+M3o+BszZoxxOBwux++1PtuunsfFixdN9erVr/ufzVsFl8byQLdu3eTl5WU9bty4saS/T1dL0pYtW/THH3+oU6dO+vPPP3XixAmdOHFCycnJat68ub777rtrnsJdunSpQkJC1LFjR6utYMGCev7553XmzBl9++23Lv0feeQRFS9e/Jo1HzlyRNu2bVNsbKz8/Pys9iZNmqhGjRpZXvenn37a5XHjxo31559/KikpyWrz8fGx/p2YmKgTJ06oSZMm2rt3rxITE7O8LEmKjY3VkSNHtGrVKqvtww8/lI+Pjx555JEbqvmzzz5Tamqq2rVrZ+2bEydOKCQkRBUrVnRZliT5+fmpQ4cO1uPKlSsrMDBQVatWVb169az2tH+nvQ5SUlK0YsUKtWnTRuXLl7f6hYaGqlOnTlqzZo213dq3b69Lly65XGJZsWKFTp06pfbt20uSjDFauHChWrduLWOMS+0xMTFKTEzUpk2bJP39GgoNDVXbtm2t+fn6+urJJ5+87vZKSUnR8uXL1aZNG5UpU8Zqr1q1qmJiYtL1v9n97XQ6rTErKSkp+vPPP+Xn56fKlStb65OZpUuXSpL69evn0v6vf/1LktJdLsoOJUuW1EMPPWQ99vf3V2xsrDZv3qz4+HhJf1+Cbty4sYoUKeKyn6Kjo5WSkqLvvvvOZZ5ZOYbTLF26VHXr1nW5zOvn56cnn3xS+/fv1/bt2136d+nSxWUfXalHjx7WvwMDA1W5cmUVKlRI7dq1s9rTXu9pr+u0vr/99pv++OOPLNWcVZGRkdb7qSQVL15clStXdlm2u9s2M2+88Yb69Omj119/XYMHD7bab+T9u1evXi6Pe/fuLen/Xp/ucuf4vXLfJicn68SJE2rQoIGMMRkOK8jIlfP466+/lJiYqMaNG1/3+LtVMFg6D1z54SBJRYoUkfT3C0iS9ebQpUuXTOeRmJhoTXe1AwcOqGLFiukGNFatWtV6/krlypW7bs1p01SoUCHdcxUqVMjyC/5a6+7v7y9JWrt2rYYOHap169alGz+UmJiogICALC1Lklq0aKHQ0FB9+OGHat68uVJTU/Xxxx/rwQcfVOHChbM0j6u3zx9//CFjjCpWrJhh/6u/aVe6dOl0984JCAhQWFhYujbp/14Hx48f19mzZ1W5cuV0y6hatapSU1N16NAhVatWTbVq1VKVKlU0f/58de/eXZI0f/58BQUFqVmzZtb8Tp06pRkzZmjGjBkZ1n7s2DFJf+/vChUqpKs7o1qudvz4cZ07dy7D7VO5cuV0b+43u79TU1P19ttva+rUqdq3b59SUlKs54oVK3bNaQ8cOCAPD490r+uQkBAFBgamO1ayQ0bbtVKlSpL+HvMTEhKiP/74Q7/88kum4SZtP6XJyjGc5sCBAy4BPM2V7w/Vq1e/7ry9vb3T1RcQEJDp6z3tdS1Jr732mh588EFVqlRJ1atX17333qvOnTurZs2aWV6PjFz9/iL9/R5z5bLd3bYZ+fbbb7VkyRK9/PLLLuOC0uYvuff+ffWxEhERIQ8Pjxu+TYA7x+/Bgwc1ZMgQLV682GU7pdWZFV9++aVGjhypLVu2uIxDyi/3DCMI5QFPT88M240xkmT9b+HNN99U7dq1M+x75VmZm5XZ//ZywvXWfc+ePWrevLmqVKmi8ePHKywsTF5eXlq6dKkmTJjg9mBGT09PderUSTNnztTUqVO1du1aHTlyJMNv9WXm6u2Tmpoqh8Ohr776KsP1uXrfZLbO19sW7mrfvr1GjRqlEydOqHDhwlq8eLE6duxo3cYgbds9/vjjmb5J3+wHkbuyY3+PHj1ar776qp544gmNGDFCRYsWlYeHh1544YUsv15u5g37yuCVXVJTU9WiRQu99NJLGT6fFpzS5OQxnNm8b+Z1fdddd2nPnj3673//qxUrVujdd9/VhAkTNG3aNJezTO7KyrLd3bYZqVatmk6dOqW5c+fqqaeecgmL2fH+nVsBIiUlRS1atNDJkyf18ssvq0qVKipUqJAOHz6srl27Zun4+f777/XAAw/orrvu0tSpUxUaGqqCBQtq1qxZ6QbJ36oIQregiIgISX+fMo+OjnZ7+rJly+qXX35Ramqqy1mhnTt3Ws/fyDwlZfjttozabtQXX3yhCxcuaPHixS7/u7v6cpM7YmNj9dZbb+mLL77QV199peLFi2d4iSarIiIiZIxRuXLlsvSmeaOKFy8uX19f7dq1K91zO3fulIeHh8tZpfbt22v48OFauHChgoODlZSU5HJJrnjx4ipcuLBSUlKu+7oqW7asfv31VxljXN6UM6olo7p9fHwyvOxx9fTu7O/MPhwWLFigpk2b6r333nNpP3XqlIKCgq5Za9myZZWamqo//vjDOiMi/X1H5lOnTrkcK0WKFNGpU6dcpr948aKOHj2apTrT7N69O912/f333yXJ+vZlRESEzpw5c0PH//WULVs209dU2vO5oWjRourWrZu6deumM2fO6K677tKwYcOuGYSyIyBkx7YNCgrSggUL1KhRIzVv3lxr1qxRyZIlrflL7r1///HHHy5havfu3UpNTXX5Nq47657V43fbtm36/fffNWfOHMXGxlrtGX17L7PlL1y4UN7e3lq+fLmcTqfVPmvWrCzXm9cYI3QLioqKUkREhMaNG6czZ86ke/748ePXnL5Vq1aKj4/X/PnzrbbLly9r0qRJ8vPzU5MmTdyuqWTJkqpevbo++OADl5q+/fZbbdu2ze35ZSbtf3RX/g8uMTHxpg6qmjVrqmbNmnr33Xe1cOFCdejQ4aZu9vjwww/L09NTw4cPT3f2xhijP//884bnfSVPT0/dc889+u9//+tyijwhIUEfffSRGjVqZF1OlP6+tFGjRg3Nnz9f8+fPV2hoqO666y6X+T3yyCNauHChfv3113TLu/J11apVKx05csTlK7Bnz57N9JLa1XXHxMTo888/18GDB632HTt2aPny5en6Slnb34UKFUoXRNLmcfV++PTTT695a4Q0rVq1kqR0Pz0xfvx4SdJ9991ntUVERKQbPzJjxox0Z4TS7jOVUa3S3+PtFi1aZD1OSkrSBx98oNq1ayskJESS1K5dO61bty7d9kqb7+XLl6+7bplp1aqV1q9fr3Xr1lltycnJmjFjhsLDwxUZGXnD886qq48RPz8/VahQ4bq3Z7jets2K7Nq2pUuX1tdff61z586pRYsW1jrdyPt32lfT00yaNEmS1LJlS6sts9d/RrJ6/GZ0/Blj9Pbbb6ebZ2bb3tPTUw6Hw+U42L9/f57eBdtdnBG6BXl4eOjdd99Vy5YtVa1aNXXr1k2lSpXS4cOHtWrVKvn7++uLL77IdPonn3xS06dPV9euXbVx40aFh4drwYIFWrt2rSZOnJjlsTFXGz16tB588EE1bNhQ3bp1019//aXJkyerevXqGR7wN+Kee+6Rl5eXWrduraeeekpnzpzRzJkzVaJEiXT/83ZHbGysdXdddy6LZSQiIkIjR47UwIEDtX//frVp00aFCxfWvn37tGjRIj355JPZdiffkSNHauXKlWrUqJGeffZZFShQQNOnT9eFCxf0xhtvpOvfvn17DRkyRN7e3urevXu6cWJjx47VqlWrVK9ePfXs2VORkZE6efKkNm3apK+//lonT56UJPXs2VOTJ09WbGysNm7cqNDQUM2dO1e+vr5Zqnv48OFatmyZGjdurGeffdYK4tWqVdMvv/xi9XNnf0dFRemdd97RyJEjVaFCBZUoUULNmjXT/fffr9dee03dunVTgwYNtG3bNn344YcuA8wzU6tWLXXp0kUzZszQqVOn1KRJE61fv15z5sxRmzZt1LRpU6tvjx499PTTT+uRRx5RixYttHXrVi1fvjzdWafatWvL09NTr7/+uhITE+V0Oq37JEl/X3rp3r27fv75ZwUHB+v9999XQkKCS/h78cUXtXjxYt1///3q2rWroqKilJycrG3btmnBggXav3//dc92ZWbAgAH6+OOP1bJlSz3//PMqWrSo5syZo3379mnhwoW5crPEyMhI3X333YqKilLRokW1YcMGLViwQM8999w1p7vets2K7Ny2FSpU0IoVK3T33XcrJiZG33zzjfz9/d1+/963b58eeOAB3XvvvVq3bp3+85//qFOnTqpVq5bVJyoqSl9//bXGjx+vkiVLqly5chmO9ZKyfvxWqVJFERER6t+/vw4fPix/f38tXLgw3VihtOVL0vPPP6+YmBh5enqqQ4cOuu+++zR+/Hjde++96tSpk44dO6YpU6aoQoUKLsf6LS13v6R2+8vK1+c//fRTl/aMvvZtjDGbN282Dz/8sClWrJhxOp2mbNmypl27diYuLu66dSQkJJhu3bqZoKAg4+XlZWrUqJFu/mnLffPNN9NNn1lN8+bNM1WqVDFOp9NUr17dLF682DzyyCOmSpUqLv2Uydfnjx8/7tJv1qxZRpLZt2+f1bZ48WJTs2ZN4+3tbcLDw83rr79u3n///XT9svL1+TRHjx41np6eplKlShlur4xkVnOahQsXmkaNGplChQqZQoUKmSpVqphevXqZXbt2udRYrVq1dNNefS+SNMrg686bNm0yMTExxs/Pz/j6+pqmTZtmeg+RP/74w0gyksyaNWsy7JOQkGB69eplwsLCTMGCBU1ISIhp3ry5mTFjhku/AwcOmAceeMD4+vqaoKAg06dPH7Ns2bIsfX3eGGO+/fZbExUVZby8vEz58uXNtGnTrG16pazu7/j4eHPfffeZwoULG0nWvj9//rz517/+ZUJDQ42Pj49p2LChWbduXbrXR2YuXbpkhg8fbsqVK2cKFixowsLCzMCBA8358+dd+qWkpJiXX37ZBAUFGV9fXxMTE2N2796d7uvzxhgzc+ZMU758eet2AWnbK22/L1++3NSsWdM4nU5TpUqVdO8Jxvz9lfaBAweaChUqGC8vLxMUFGQaNGhgxo0bZ90X6lrHsDEZv56M+fuePG3btjWBgYHG29vb1K1bN909XzJ7vzLm76/PFypUKF17Vl/vI0eONHXr1jWBgYHGx8fHVKlSxYwaNcpar2u53rbNqKarXwdZ2baZyWg5P/30k3VLi7Svkmfl/TvteNi+fbtp27atKVy4sClSpIh57rnnzLlz51yWsXPnTnPXXXcZHx8fI+m6X6XP6vG7fft2Ex0dbfz8/ExQUJDp2bOn2bp1a7r30suXL5vevXub4sWLG4fD4XIcv/fee6ZixYrW63nWrFkZHuu3KocxNzgyE/j/ateureLFi1/zrrB57cSJEwoNDdWQIUP06quv5nU5sKnw8HBVr17d5U7fsK9hw4Zp+PDhOn78+A2f4cPNY4wQsuzSpUvprp+vXr1aW7dutX7u4FY1e/ZspaSkqHPnznldCgDgFsIYIWTZ4cOHFR0drccff1wlS5bUzp07NW3aNIWEhKS76eCt4ptvvtH27ds1atQotWnTxna/WA0AuDaCELKsSJEiioqK0rvvvqvjx4+rUKFCuu+++zR27Njr3rgur7z22mv64Ycf1LBhQ+ubGAAApMnTMULfffed3nzzTW3cuFFHjx7VokWL1KZNm2tOs3r1avXr10+//fabwsLCNHjw4Ov+Ci8AAEBG8nSMUHJysmrVqpXuHgqZ2bdvn+677z41bdpUW7Zs0QsvvKAePXpkeD8IAACA67llvjXmcDiue0bo5Zdf1pIlS1xuBtehQwedOnVKy5Yty4UqAQDA7SRfjRFat25duluWx8TE6IUXXsh0mgsXLrjcrTQ1NVUnT55UsWLF8s0PwgEAYHfGGJ0+fVolS5bM1ht/5qsgFB8fr+DgYJe2tN9UOnfuXIY/DjhmzBgNHz48t0oEAAA56NChQypdunS2zS9fBaEbMXDgQPXr1896nJiYqDJlyujQoUMuv9MEAABuXUlJSQoLC7vhn4nKTL4KQiEhIUpISHBpS0hIkL+/f4ZngyTJ6XS6/CJuGn9/f4IQAAD5THYPa8lXd5auX7++4uLiXNpWrlyp+vXr51FFAAAgP8vTIHTmzBlt2bJFW7ZskfT31+O3bNmigwcPSvr7slZsbKzV/+mnn9bevXv10ksvaefOnZo6dao++eQT9e3bNy/KBwAA+VyeBqENGzbojjvu0B133CFJ6tevn+644w4NGTJEknT06FErFElSuXLltGTJEq1cuVK1atXSW2+9pXfffVcxMTF5Uj8AAMjfbpn7COWWpKQkBQQEKDExkTFCAADkopSUFF26dCnT5728vDL9anxOfX7nq8HSAAAg/zHGKD4+XqdOnbpmPw8PD5UrV05eXl65U5gIQgAAIIelhaASJUrI19c3w29+paam6siRIzp69KjKlCmTazc9JggBAIAck5KSYoWgYsWKXbNv8eLFdeTIEV2+fFkFCxbMlfry1dfnAQBA/pI2JsjX1/e6fdMuiaWkpORoTVciCAEAgByXlUtdefEboAQhAABgWwQhAABgWwQhAABgWwQhAACQ47Jy/+a8uMczQQgAAOSYtK/Bnz179rp9L168KEny9PTM0ZquxH2EAABAjvH09FRgYKCOHTsmSde8oeLx48fl6+urAgVyL54QhAAAQI4KCQmRJCsMZcbDwyNX7yotEYQAAEAOczgcCg0NVYkSJW74R1dzCkEIAADkCk9Pz1wd/5MVDJYGAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2ledBaMqUKQoPD5e3t7fq1aun9evXX7P/xIkTVblyZfn4+CgsLEx9+/bV+fPnc6laAABwO8nTIDR//nz169dPQ4cO1aZNm1SrVi3FxMTo2LFjGfb/6KOPNGDAAA0dOlQ7duzQe++9p/nz5+uVV17J5coBAMDtIE+D0Pjx49WzZ09169ZNkZGRmjZtmnx9ffX+++9n2P+HH35Qw4YN1alTJ4WHh+uee+5Rx44dr3sWCQAAICN5FoQuXryojRs3Kjo6+v+K8fBQdHS01q1bl+E0DRo00MaNG63gs3fvXi1dulStWrXKdDkXLlxQUlKSyx8AAIAkFcirBZ84cUIpKSkKDg52aQ8ODtbOnTsznKZTp046ceKEGjVqJGOMLl++rKeffvqal8bGjBmj4cOHZ2vtAADg9pDng6XdsXr1ao0ePVpTp07Vpk2b9Nlnn2nJkiUaMWJEptMMHDhQiYmJ1t+hQ4dysWIAAHAry7MzQkFBQfL09FRCQoJLe0JCgkJCQjKc5tVXX1Xnzp3Vo0cPSVKNGjWUnJysJ598UoMGDZKHR/pc53Q65XQ6s38FAABAvpdnZ4S8vLwUFRWluLg4qy01NVVxcXGqX79+htOcPXs2Xdjx9PSUJBljcq5YAABwW8qzM0KS1K9fP3Xp0kV16tRR3bp1NXHiRCUnJ6tbt26SpNjYWJUqVUpjxoyRJLVu3Vrjx4/XHXfcoXr16mn37t169dVX1bp1aysQAQAAZFWeBqH27dvr+PHjGjJkiOLj41W7dm0tW7bMGkB98OBBlzNAgwcPlsPh0ODBg3X48GEVL15crVu31qhRo/JqFQAAQD7mMDa7ppSUlKSAgAAlJibK398/r8sBAABZkFOf3/nqW2MAAADZiSAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsK8+D0JQpUxQeHi5vb2/Vq1dP69evv2b/U6dOqVevXgoNDZXT6VSlSpW0dOnSXKoWAADcTgrk5cLnz5+vfv36adq0aapXr54mTpyomJgY7dq1SyVKlEjX/+LFi2rRooVKlCihBQsWqFSpUjpw4IACAwNzv3gAAJDvOYwxJq8WXq9ePd15552aPHmyJCk1NVVhYWHq3bu3BgwYkK7/tGnT9Oabb2rnzp0qWLDgDS0zKSlJAQEBSkxMlL+//03VDwAAckdOfX7n2aWxixcvauPGjYqOjv6/Yjw8FB0drXXr1mU4zeLFi1W/fn316tVLwcHBql69ukaPHq2UlJRMl3PhwgUlJSW5/AEAAEh5GIROnDihlJQUBQcHu7QHBwcrPj4+w2n27t2rBQsWKCUlRUuXLtWrr76qt956SyNHjsx0OWPGjFFAQID1FxYWlq3rAQAA8q88HyztjtTUVJUoUUIzZsxQVFSU2rdvr0GDBmnatGmZTjNw4EAlJiZaf4cOHcrFigEAwK0szwZLBwUFydPTUwkJCS7tCQkJCgkJyXCa0NBQFSxYUJ6enlZb1apVFR8fr4sXL8rLyyvdNE6nU06nM3uLBwAAt4UbOiP0/fff6/HHH1f9+vV1+PBhSdLcuXO1Zs2aLM/Dy8tLUVFRiouLs9pSU1MVFxen+vXrZzhNw4YNtXv3bqWmplptv//+u0JDQzMMQQAAANfidhBauHChYmJi5OPjo82bN+vChQuSpMTERI0ePdqtefXr108zZ87UnDlztGPHDj3zzDNKTk5Wt27dJEmxsbEaOHCg1f+ZZ57RyZMn1adPH/3+++9asmSJRo8erV69erm7GgAAAO5fGhs5cqSmTZum2NhYzZs3z2pv2LDhNQctZ6R9+/Y6fvy4hgwZovj4eNWuXVvLli2zBlAfPHhQHh7/l9XCwsK0fPly9e3bVzVr1lSpUqXUp08fvfzyy+6uBgAAgPv3EfL19dX27dsVHh6uwoULa+vWrSpfvrz27t2ryMhInT9/PqdqzRbcRwgAgPznlrmPUEhIiHbv3p2ufc2aNSpfvny2FAUAAJAb3A5CPXv2VJ8+ffTTTz/J4XDoyJEj+vDDD9W/f38988wzOVEjAABAjnB7jNCAAQOUmpqq5s2b6+zZs7rrrrvkdDrVv39/9e7dOydqBAAAyBFujRFKSUnR2rVrVbNmTfn6+mr37t06c+aMIiMj5efnl5N1ZhvGCAEAkP/k1Oe3W2eEPD09dc8992jHjh0KDAxUZGRkthUCAACQ29weI1S9enXt3bs3J2oBAADIVW4HoZEjR6p///768ssvdfToUX7ZHQAA5Ftu30foyhscOhwO69/GGDkcDqWkpGRfdTmAMUIAAOQ/t8QYIUlatWpVti0cAAAgL7kdhJo0aZITdQAAAOQ6t4OQJJ06dUrvvfeeduzYIUmqVq2annjiCQUEBGRrcQAAADnJ7cHSGzZsUEREhCZMmKCTJ0/q5MmTGj9+vCIiIrRp06acqBEAACBHuD1YunHjxqpQoYJmzpypAgX+PqF0+fJl9ejRQ3v37tV3332XI4VmFwZLAwCQ/+TU57fbQcjHx0ebN29WlSpVXNq3b9+uOnXq6OzZs9lWXE4gCAEAkP/cMr8+7+/vr4MHD6ZrP3TokAoXLpwtRQEAAOQGt4NQ+/bt1b17d82fP1+HDh3SoUOHNG/ePPXo0UMdO3bMiRoBAAByhNvfGhs3bpwcDodiY2N1+fJlSVLBggX1zDPPaOzYsdleIAAAQE5xe4xQmrNnz2rPnj2SpIiICPn6+mZrYTmFMUIAAOQ/t8ydpRMTE5WSkqKiRYuqRo0aVvvJkydVoEABwgUAAMg33B4j1KFDB82bNy9d+yeffKIOHTpkS1EAAAC5we0g9NNPP6lp06bp2u+++2799NNP2VIUAABAbnA7CF24cMEaJH2lS5cu6dy5c9lSFAAAQG5wOwjVrVtXM2bMSNc+bdo0RUVFZUtRAAAAucHtwdIjR45UdHS0tm7dqubNm0uS4uLi9PPPP2vFihXZXiAAAEBOcfuMUMOGDbVu3TqFhYXpk08+0RdffKEKFSrol19+UePGjXOiRgAAgBxxw/cRyq+4jxAAAPlPnt9H6PLly0pJSZHT6bTaEhISNG3aNCUnJ+uBBx5Qo0aNsq0wAACAnJblINSzZ095eXlp+vTpkqTTp0/rzjvv1Pnz5xUaGqoJEybov//9r1q1apVjxQIAAGSnLI8RWrt2rR555BHr8QcffKCUlBT98ccf2rp1q/r166c333wzR4oEAADICVkOQocPH1bFihWtx3FxcXrkkUcUEBAgSerSpYt+++237K8QAAAgh2Q5CHl7e7vcMPHHH39UvXr1XJ4/c+ZM9lYHAACQg7IchGrXrq25c+dKkr7//nslJCSoWbNm1vN79uxRyZIls79CAACAHJLlwdJDhgxRy5Yt9cknn+jo0aPq2rWrQkNDrecXLVqkhg0b5kiRAAAAOSHLQahJkybauHGjVqxYoZCQED366KMuz9euXVt169bN9gIBAAByCjdUBAAAt7yc+vx2+yc2AAAAbhcEIQAAYFsEIQAAYFsEIQAAYFsEIQAAYFtZ/vp8miJFisjhcKRrdzgc8vb2VoUKFdS1a1d169YtWwoEAADIKW4HoSFDhmjUqFFq2bKldd+g9evXa9myZerVq5f27dunZ555RpcvX1bPnj2zvWAAAIDs4nYQWrNmjUaOHKmnn37apX369OlasWKFFi5cqJo1a+rf//43QQgAANzS3B4jtHz5ckVHR6drb968uZYvXy5JatWqlfbu3Xvz1QEAAOQgt4NQ0aJF9cUXX6Rr/+KLL1S0aFFJUnJysgoXLnzz1QEAAOQgty+Nvfrqq3rmmWe0atUqa4zQzz//rKVLl2ratGmSpJUrV6pJkybZWykAAEA2u6HfGlu7dq0mT56sXbt2SZIqV66s3r17q0GDBtleYHbjt8YAAMh/curzmx9dBQAAt7yc+vx2+9JYUlJShu0Oh0NOp1NeXl43XRQAAEBucDsIBQYGZnhDxTSlS5dW165dNXToUHl4cONqAABw63I7CM2ePVuDBg1S165dXW6oOGfOHA0ePFjHjx/XuHHj5HQ69corr2R7wQAAANnF7SA0Z84cvfXWW2rXrp3V1rp1a9WoUUPTp09XXFycypQpo1GjRhGEAADALc3ta1c//PCD7rjjjnTtd9xxh9atWydJatSokQ4ePHjz1QEAAOQgt4NQWFiY3nvvvXTt7733nsLCwiRJf/75p4oUKXLz1QEAAOQgty+NjRs3To8++qi++uor3XnnnZKkDRs2aOfOnVqwYIGkv2+w2L59++ytFAAAIJvd0H2E9u/fr+nTp7vcUPGpp55SeHh4dteX7biPEAAA+U++uKHir7/+qurVq2fX7HIEQQgAgPwnpz6/b/pGP6dPn9aMGTNUt25d1apVKztqAgAAyBU3HIS+++47denSRaGhoRo3bpyaNWumH3/8MTtrAwAAyFFuDZaOj4/X7Nmz9d577ykpKUnt2rXThQsX9PnnnysyMjKnagQAAMgRWT4j1Lp1a1WuXFm//PKLJk6cqCNHjmjSpEk5WRsAAECOyvIZoa+++krPP/+8nnnmGVWsWDEnawIAAMgVWT4jtGbNGp0+fVpRUVGqV6+eJk+erBMnTuRkbQAAADkqy0Hon//8p2bOnKmjR4/qqaee0rx581SyZEmlpqZq5cqVOn369A0XMWXKFIWHh8vb21v16tXT+vXrszTdvHnz5HA41KZNmxteNgAAsC+3vzVWqFAhPfHEE1qzZo22bdumf/3rXxo7dqxKlCihBx54wO0C5s+fr379+mno0KHatGmTatWqpZiYGB07duya0+3fv1/9+/dX48aN3V4mAACAdJP3EapcubLeeOMN/e9//9PHH398Q/MYP368evbsqW7duikyMlLTpk2Tr6+v3n///UynSUlJ0WOPPabhw4erfPnyN1o+AACwuZu+oaIkeXp6qk2bNlq8eLFb0128eFEbN25UdHT0/xXk4aHo6Gjrl+wz8tprr6lEiRLq3r37dZdx4cIFJSUlufwBAABI2RSEbtSJEyeUkpKi4OBgl/bg4GDFx8dnOM2aNWv03nvvaebMmVlaxpgxYxQQEGD9hYWF3XTdAADg9pCnQchdp0+fVufOnTVz5kwFBQVlaZqBAwcqMTHR+jt06FAOVwkAAPILt+4snd2CgoLk6emphIQEl/aEhASFhISk679nzx7t379frVu3ttpSU1MlSQUKFNCuXbsUERHhMo3T6ZTT6cyB6gEAQH6Xp2eEvLy8FBUVpbi4OKstNTVVcXFxql+/frr+VapU0bZt27Rlyxbr74EHHlDTpk21ZcsWLnsBAAC35OkZIUnq16+funTpojp16qhu3bqaOHGikpOT1a1bN0lSbGysSpUqpTFjxsjb21vVq1d3mT4wMFCS0rUDAABcT54Hofbt2+v48eMaMmSI4uPjVbt2bS1btswaQH3w4EF5eOSroUwAACCfcBhjTF4XkZuSkpIUEBCgxMRE+fv753U5AAAgC3Lq85tTLQAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLZuiSA0ZcoUhYeHy9vbW/Xq1dP69esz7Ttz5kw1btxYRYoUUZEiRRQdHX3N/gAAAJnJ8yA0f/589evXT0OHDtWmTZtUq1YtxcTE6NixYxn2X716tTp27KhVq1Zp3bp1CgsL0z333KPDhw/ncuUAACC/cxhjTF4WUK9ePd15552aPHmyJCk1NVVhYWHq3bu3BgwYcN3pU1JSVKRIEU2ePFmxsbHX7Z+UlKSAgAAlJibK39//pusHAAA5L6c+v/P0jNDFixe1ceNGRUdHW20eHh6Kjo7WunXrsjSPs2fP6tKlSypatGiGz1+4cEFJSUkufwAAAFIeB6ETJ04oJSVFwcHBLu3BwcGKj4/P0jxefvlllSxZ0iVMXWnMmDEKCAiw/sLCwm66bgAAcHvI8zFCN2Ps2LGaN2+eFi1aJG9v7wz7DBw4UImJidbfoUOHcrlKAABwqyqQlwsPCgqSp6enEhISXNoTEhIUEhJyzWnHjRunsWPH6uuvv1bNmjUz7ed0OuV0OrOlXgAAcHvJ0zNCXl5eioqKUlxcnNWWmpqquLg41a9fP9Pp3njjDY0YMULLli1TnTp1cqNUAABwG8rTM0KS1K9fP3Xp0kV16tRR3bp1NXHiRCUnJ6tbt26SpNjYWJUqVUpjxoyRJL3++usaMmSIPvroI4WHh1tjifz8/OTn55dn6wEAAPKfPA9C7du31/HjxzVkyBDFx8erdu3aWrZsmTWA+uDBg/Lw+L8TV++8844uXryotm3busxn6NChGjZsWG6WDgAA8rk8v49QbuM+QgAA5D+35X2EAAAA8hJBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2NYtEYSmTJmi8PBweXt7q169elq/fv01+3/66aeqUqWKvL29VaNGDS1dujSXKgUAALeTPA9C8+fPV79+/TR06FBt2rRJtWrVUkxMjI4dO5Zh/x9++EEdO3ZU9+7dtXnzZrVp00Zt2rTRr7/+msuVAwCA/M5hjDF5WUC9evV05513avLkyZKk1NRUhYWFqXfv3howYEC6/u3bt1dycrK+/PJLq+2f//ynateurWnTpl13eUlJSQoICFBiYqL8/f2zb0UAAECOyanP7zw9I3Tx4kVt3LhR0dHRVpuHh4eio6O1bt26DKdZt26dS39JiomJybQ/AABAZgrk5cJPnDihlJQUBQcHu7QHBwdr586dGU4THx+fYf/4+PgM+1+4cEEXLlywHicmJkr6O1kCAID8Ie1zO7svZOVpEMoNY8aM0fDhw9O1h4WF5UE1AADgZvz5558KCAjItvnlaRAKCgqSp6enEhISXNoTEhIUEhKS4TQhISFu9R84cKD69etnPT516pTKli2rgwcPZuuGhPuSkpIUFhamQ4cOMV7rFsD+uHWwL24d7ItbR2JiosqUKaOiRYtm63zzNAh5eXkpKipKcXFxatOmjaS/B0vHxcXpueeey3Ca+vXrKy4uTi+88ILVtnLlStWvXz/D/k6nU06nM117QEAAL+pbhL+/P/viFsL+uHWwL24d7Itbh4dH9g5vzvNLY/369VOXLl1Up04d1a1bVxMnTlRycrK6desmSYqNjVWpUqU0ZswYSVKfPn3UpEkTvfXWW7rvvvs0b948bdiwQTNmzMjL1QAAAPlQngeh9u3b6/jx4xoyZIji4+NVu3ZtLVu2zBoQffDgQZf016BBA3300UcaPHiwXnnlFVWsWFGff/65qlevnlerAAAA8qk8D0KS9Nxzz2V6KWz16tXp2h599FE9+uijN7Qsp9OpoUOHZni5DLmLfXFrYX/cOtgXtw72xa0jp/ZFnt9QEQAAIK/k+U9sAAAA5BWCEAAAsC2CEAAAsC2CEAAAsK3bMghNmTJF4eHh8vb2Vr169bR+/fpr9v/0009VpUoVeXt7q0aNGlq6dGkuVXr7c2dfzJw5U40bN1aRIkVUpEgRRUdHX3ffwT3uHhtp5s2bJ4fDYd34FDfP3X1x6tQp9erVS6GhoXI6napUqRLvVdnE3X0xceJEVa5cWT4+PgoLC1Pfvn11/vz5XKr29vXdd9+pdevWKlmypBwOhz7//PPrTrN69Wr94x//kNPpVIUKFTR79mz3F2xuM/PmzTNeXl7m/fffN7/99pvp2bOnCQwMNAkJCRn2X7t2rfH09DRvvPGG2b59uxk8eLApWLCg2bZtWy5Xfvtxd1906tTJTJkyxWzevNns2LHDdO3a1QQEBJj//e9/uVz57cnd/ZFm3759plSpUqZx48bmwQcfzJ1ib3Pu7osLFy6YOnXqmFatWpk1a9aYffv2mdWrV5stW7bkcuW3H3f3xYcffmicTqf58MMPzb59+8zy5ctNaGio6du3by5XfvtZunSpGTRokPnss8+MJLNo0aJr9t+7d6/x9fU1/fr1M9u3bzeTJk0ynp6eZtmyZW4t97YLQnXr1jW9evWyHqekpJiSJUuaMWPGZNi/Xbt25r777nNpq1evnnnqqadytE47cHdfXO3y5cumcOHCZs6cOTlVoq3cyP64fPmyadCggXn33XdNly5dCELZxN198c4775jy5cubixcv5laJtuHuvujVq5dp1qyZS1u/fv1Mw4YNc7ROu8lKEHrppZdMtWrVXNrat29vYmJi3FrWbXVp7OLFi9q4caOio6OtNg8PD0VHR2vdunUZTrNu3TqX/pIUExOTaX9kzY3si6udPXtWly5dyvYf2LOjG90fr732mkqUKKHu3bvnRpm2cCP7YvHixapfv7569eql4OBgVa9eXaNHj1ZKSkpulX1bupF90aBBA23cuNG6fLZ3714tXbpUrVq1ypWa8X+y6/P7lrizdHY5ceKEUlJSrJ/nSBMcHKydO3dmOE18fHyG/ePj43OsTju4kX1xtZdfflklS5ZM90KH+25kf6xZs0bvvfeetmzZkgsV2seN7Iu9e/fqm2++0WOPPaalS5dq9+7devbZZ3Xp0iUNHTo0N8q+Ld3IvujUqZNOnDihRo0ayRijy5cv6+mnn9Yrr7ySGyXjCpl9ficlJencuXPy8fHJ0nxuqzNCuH2MHTtW8+bN06JFi+Tt7Z3X5djO6dOn1blzZ82cOVNBQUF5XY7tpaamqkSJEpoxY4aioqLUvn17DRo0SNOmTcvr0mxn9erVGj16tKZOnapNmzbps88+05IlSzRixIi8Lg036LY6IxQUFCRPT08lJCS4tCckJCgkJCTDaUJCQtzqj6y5kX2RZty4cRo7dqy+/vpr1axZMyfLtA1398eePXu0f/9+tW7d2mpLTU2VJBUoUEC7du1SREREzhZ9m7qRYyM0NFQFCxaUp6en1Va1alXFx8fr4sWL8vLyytGab1c3si9effVVde7cWT169JAk1ahRQ8nJyXryySc1aNAglx8JR87K7PPb398/y2eDpNvsjJCXl5eioqIUFxdntaWmpiouLk7169fPcJr69eu79JeklStXZtofWXMj+0KS3njjDY0YMULLli1TnTp1cqNUW3B3f1SpUkXbtm3Tli1brL8HHnhATZs21ZYtWxQWFpab5d9WbuTYaNiwoXbv3m2FUUn6/fffFRoaSgi6CTeyL86ePZsu7KQFVMNPd+aqbPv8dm8c961v3rx5xul0mtmzZ5vt27ebJ5980gQGBpr4+HhjjDGdO3c2AwYMsPqvXbvWFChQwIwbN87s2LHDDB06lK/PZxN398XYsWONl5eXWbBggTl69Kj1d/r06bxahduKu/vjanxrLPu4uy8OHjxoChcubJ577jmza9cu8+WXX5oSJUqYkSNH5tUq3Dbc3RdDhw41hQsXNh9//LHZu3evWbFihYmIiDDt2rXLq1W4bZw+fdps3rzZbN682Ugy48ePN5s3bzYHDhwwxhgzYMAA07lzZ6t/2tfnX3zxRbNjxw4zZcoUvj6fZtKkSaZMmTLGy8vL1K1b1/z444/Wc02aNDFdunRx6f/JJ5+YSpUqGS8vL1OtWjWzZMmSXK749uXOvihbtqyRlO5v6NChuV/4bcrdY+NKBKHs5e6++OGHH0y9evWM0+k05cuXN6NGjTKXL1/O5apvT+7si0uXLplhw4aZiIgI4+3tbcLCwsyzzz5r/vrrr9wv/DazatWqDD8D0rZ/ly5dTJMmTdJNU7t2bePl5WXKly9vZs2a5fZyHcZwLg8AANjTbTVGCAAAwB0EIQAAYFsEIQAAYFsEIQAAYFsEIQAAYFsEIQAAYFsEIQAAYFsEIeA2FR4erokTJ2a5/+rVq+VwOHTq1Kkcq+lmzJ49W4GBgXldRjpxcXGqWrWqUlJSJEnDhg1T7dq1rzlN165d1aZNG+vx3XffrRdeeMF6fPW+czgc+vzzz7Ov6Ovo0KGD3nrrrVxbHpCXCEJAHnM4HNf8GzZs2A3N9+eff9aTTz6Z5f4NGjTQ0aNHFRAQcEPLy6pbNdDcqJdeekmDBw92+UHU63n77bc1e/bsLPc/evSoWrZseQPV3ZjBgwdr1KhRSkxMzLVlAnnltvr1eSA/Onr0qPXv+fPna8iQIdq1a5fV5ufnZ/3bGKOUlBQVKHD9Q7d48eJu1eHl5ZXpL24jY2vWrNGePXv0yCOPuDWdu2Ezt/dL9erVFRERof/85z/q1atXri4byG2cEQLyWEhIiPUXEBAgh8NhPd65c6cKFy6sr776SlFRUXI6ndaH74MPPqjg4GD5+fnpzjvv1Ndff+0y34wur7z77rt66KGH5Ovrq4oVK2rx4sXW81dfGks7c7N8+XJVrVpVfn5+uvfee12C2+XLl/X8888rMDBQxYoV08svv6wuXbq4XPa50urVq9WtWzclJiamO+P1119/KTY2VkWKFJGvr69atmypP/74I9Ptdvz4cdWpU0cPPfSQLly4oNTUVI0ZM0blypWTj4+PatWqpQULFqRbv7i4ONWpU0e+vr5q0KCBS+jcunWrmjZtqsKFC8vf319RUVHasGFDpjXMmzdPLVq0kLe3d7rnpk+frrCwMPn6+qpdu3YuZ1euvjR2PVdfGtu2bZuaNWsmHx8fFStWTE8++aTOnDmTbv7jxo1TaGioihUrpl69eunSpUtWn6lTp6pixYry9vZWcHCw2rZt67LM1q1ba968eVmuEcivCEJAPjBgwACNHTtWO3bsUM2aNXXmzBm1atVKcXFx2rx5s+699161bt1aBw8evOZ8hg8frnbt2umXX35Rq1at9Nhjj+nkyZOZ9j979qzGjRunuXPn6rvvvtPBgwfVv39/6/nXX39dH374oWbNmqW1a9cqKSnpmmNZGjRooIkTJ8rf319Hjx7V0aNHrfl17dpVGzZs0OLFi7Vu3ToZY9SqVSuXD+80hw4dUuPGjVW9enUtWLBATqdTY8aM0QcffKBp06bpt99+U9++ffX444/r22+/dZl20KBBeuutt7RhwwYVKFBATzzxhPXcY489ptKlS+vnn3/Wxo0bNWDAABUsWDDT9fn+++9Vp06ddO27d+/WJ598oi+++ELLli3T5s2b9eyzz2Y6H3ckJycrJiZGRYoU0c8//6xPP/1UX3/9tZ577jmXfqtWrdKePXu0atUqzZkzR7Nnz7Yux23YsEHPP/+8XnvtNe3atUvLli3TXXfd5TJ93bp1tX79el24cCFb6gZuWTf3W7EAstOsWbNMQECA9Tjt15g///zz605brVo1M2nSJOtx2bJlzYQJE6zHkszgwYOtx2fOnDGSzFdffeWyrLRf0Z41a5aRZHbv3m1NM2XKFBMcHGw9Dg4ONm+++ab1+PLly6ZMmTLX/JX6q9fRGGN+//13I8msXbvWajtx4oTx8fExn3zyict0O3fuNGFhYeb55583qampxhhjzp8/b3x9fc0PP/zgMt/u3bubjh07uqzf119/bT2/ZMkSI8mcO3fOGGNM4cKFzezZszOt/WoBAQHmgw8+cGkbOnSo8fT0NP/73/+stq+++sp4eHiYo0ePGmP+/hXtK7dRkyZNTJ8+fazHGe27RYsWGWOMmTFjhilSpIg5c+aMy3p4eHiY+Ph4a/5ly5Z1+XX6Rx991LRv394YY8zChQuNv7+/SUpKynTdtm7daiSZ/fv3Z21jAPkUZ4SAfODqsw5nzpxR//79VbVqVQUGBsrPz087duy47hmhmjVrWv8uVKiQ/P39dezYsUz7+/r6KiIiwnocGhpq9U9MTFRCQoLq1q1rPe/p6amoqCi31k2SduzYoQIFCqhevXpWW7FixVS5cmXt2LHDajt37pwaN26shx9+WG+//bYcDoekv8/AnD17Vi1atJCfn5/198EHH2jPnj2ZboPQ0FBJstapX79+6tGjh6KjozV27Nh0017t3LlzGV4WK1OmjEqVKmU9rl+/vlJTU10uw92oHTt2qFatWipUqJDV1rBhw3Tzr1atmssA7iv3XYsWLVS2bFmVL19enTt31ocffqizZ8+6LMfHx0eS0rUDtxuCEJAPXPmhJ0n9+/fXokWLNHr0aH3//ffasmWLatSooYsXL15zPldf5nE4HEpNTXWrvzHGzeqzj9PpVHR0tL788ksdPnzYak8bH7NkyRJt2bLF+tu+fbvLOCHJdZ3SglTaNhg2bJh+++033Xffffrmm28UGRmpRYsWZVpPUFCQ/vrrr2xbv+x0rX1duHBhbdq0SR9//LFCQ0M1ZMgQ1apVy+XWCWmXTN0ddA/kNwQhIB9au3atunbtqoceekg1atRQSEiI9u/fn6s1BAQEKDg4WD///LPVlpKSok2bNl1zOi8vL+ueO2mqVq2qy5cv66effrLa/vzzT+3atUuRkZFWm4eHh+bOnauoqCg1bdpUR44ckSRFRkbK6XTq4MGDqlChgstfWFiYW+tVqVIl9e3bVytWrNDDDz+sWbNmZdr3jjvu0Pbt29O1Hzx40KpNkn788Ud5eHiocuXKbtWSkapVq2rr1q1KTk622tauXev2/AsUKKDo6Gi98cYb+uWXX7R//35988031vO//vqrSpcuraCgoJuuGbiVEYSAfKhixYr67LPPtGXLFm3dulWdOnW65pmdnNK7d2+NGTNG//3vf7Vr1y716dNHf/31l3WmJSPh4eE6c+aM4uLidOLECZ09e1YVK1bUgw8+qJ49e2rNmjXaunWrHn/8cZUqVUoPPvigy/Senp768MMPVatWLTVr1kzx8fEqXLiw+vfvr759+2rOnDnas2ePNm3apEmTJmnOnDlZWpdz587pueee0+rVq3XgwAGtXbtWP//8s6pWrZrpNDExMVqzZk26dm9vb3Xp0kVbt27V999/r+eff17t2rXLlq/BP/bYY9b8f/31V61atUq9e/dW586dFRwcnKV5fPnll/r3v/+tLVu26MCBA/rggw+UmprqEqS+//573XPPPTddL3CrIwgB+dD48eNVpEgRNWjQQK1bt1ZMTIz+8Y9/5HodL7/8sjp27KjY2FjVr19ffn5+iomJyXDcTJoGDRro6aefVvv27VW8eHG98cYbkqRZs2YpKipK999/v+rXry9jjJYuXZrht7YKFCigjz/+WNWqVVOzZs107NgxjRgxQq+++qrGjBmjqlWr6t5779WSJUtUrly5LK2Lp6en/vzzT8XGxqpSpUpq166dWrZsqeHDh2c6zWOPPabffvst3difChUq6OGHH1arVq10zz33qGbNmpo6dWqW6rgeX19fLV++XCdPntSdd96ptm3bqnnz5po8eXKW5xEYGKjPPvtMzZo1U9WqVTVt2jRre0rS+fPn9fnnn6tnz57ZUjNwK3OYvLzgD+C2kpqaqqpVq6pdu3YaMWJEXpeTK1588UUlJSVp+vTpeV1KtnnnnXe0aNEirVixIq9LAXIcZ4QA3LADBw5o5syZ+v3337Vt2zY988wz2rdvnzp16pTXpeWaQYMGqWzZsnlyaTKnFCxYUJMmTcrrMoBcwRkhADfs0KFD6tChg3799VcZY1S9enWNHTs23c35AOBWRRACAAC2xaUxAABgWwQhAABgWwQhAABgWwQhAABgWwQhAABgWwQhAABgWwQhAABgWwQhAABgWwQhAABgW/8PqN0VEjG6mPUAAAAASUVORK5CYII=",
+      "text/plain": [
+       "<Figure size 640x480 with 1 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "import json\n",
+    "import os\n",
+    "from matplotlib import pyplot as plt\n",
+    "metrics = ['agg_score', 'commonsense_qa/acc_norm', 'hellaswag/acc_norm', 'openbookqa/acc_norm', 'piqa/acc_norm',\n",
+    "                   'siqa/acc_norm', 'winogrande/acc_norm', 'arc/acc_norm', 'mmlu/acc_norm']\n",
+    "\n",
+    "def normalize_runname(runname):\n",
+    "    return runname.replace(\"/\", \"_\")\n",
+    "\n",
+    "grouped = (\n",
+    "    df.groupby([\"runname\", \"steps\"])\n",
+    "    .agg(\n",
+    "        {\n",
+    "            key: \"mean\" for key in metrics\n",
+    "        }\n",
+    "    )\n",
+    "    .reset_index()\n",
+    ")\n",
+    "\n",
+    "file_id=\"../assets/data/plots/removed_data_dedup\"\n",
+    "files = {}\n",
+    "for metric in metrics:\n",
+    "    datas = {}\n",
+    "    for name, group in grouped.groupby(\"runname\"):\n",
+    "        group = group[[\"steps\", metric]].sort_values(by=\"steps\")\n",
+    "        group = group.set_index(\"steps\")\n",
+    "        rolling_avg = group\n",
+    "        # rolling_avg = group.rolling(window=5).mean()\n",
+    "        datas[name] = {\n",
+    "            \"x\": (rolling_avg.index * 2048 * 1024 * 1e-9).tolist(),\n",
+    "            \"y\": rolling_avg[metric].tolist(),\n",
+    "            \"label\": runs_mapping[name],\n",
+    "        }\n",
+    "    # Sort the datata based on the steps\n",
+    "    datas = {k: v for k, v in sorted(datas.items(), key=lambda x: -x[1][\"y\"][-1])}\n",
+    "    # Create a folder\n",
+    "    os.makedirs(f\"{file_id}\", exist_ok=True)\n",
+    "    with open(f\"{file_id}/{normalize_runname(metric)}.json\", \"w\") as f:\n",
+    "        json.dump({\n",
+    "            \"data\": datas,\n",
+    "            \"layout\": {\n",
+    "                \"title\": {\n",
+    "                    \"text\": \"The originally removed data outperforms the kept data\"\n",
+    "                },\n",
+    "            }\n",
+    "        }, f)\n",
+    "    files[metric] = {\"file\": f\"{normalize_runname(metric)}.json\"}\n",
+    "# Create index\n",
+    "with open(f\"{file_id}/index.json\", \"w\") as f:\n",
+    "    json.dump({\n",
+    "        \"files\": files,\n",
+    "        \"settings\": {\n",
+    "            \"defaultMetric\": \"agg_score\",\n",
+    "            \"slider\":{\"min\":0,\"max\":10,\"default\":0}\n",
+    "        }\n",
+    "    }, f)\n",
+    "        \n",
+    "\n",
+    "# Add labels and legend\n",
+    "plt.xlabel(\"Training tokens (billions)\")\n",
+    "plt.ylabel(\"Agg Score\")\n",
+    "plt.title(\"The originally removed data outperforms the kept data\")\n",
+    "plt.legend()\n",
+    "\n",
+    "# Show the plot\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "af28ebbd054cdc33",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-04-30T12:52:05.836260Z",
+     "start_time": "2024-04-30T12:52:05.834381Z"
+    },
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/plot_wet_comparison.ipynb

@@ -0,0 +1,540 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "138889b92720ce2e",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-13T15:30:52.864251Z",
+     "start_time": "2024-05-13T15:30:52.316016Z"
+    },
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>runname</th>\n",
+       "      <th>seed</th>\n",
+       "      <th>steps</th>\n",
+       "      <th>agg_score</th>\n",
+       "      <th>commonsense_qa/acc</th>\n",
+       "      <th>commonsense_qa/acc_norm</th>\n",
+       "      <th>hellaswag/acc</th>\n",
+       "      <th>hellaswag/acc_norm</th>\n",
+       "      <th>openbookqa/acc</th>\n",
+       "      <th>openbookqa/acc_norm</th>\n",
+       "      <th>...</th>\n",
+       "      <th>siqa/acc</th>\n",
+       "      <th>siqa/acc_norm</th>\n",
+       "      <th>winogrande/acc</th>\n",
+       "      <th>winogrande/acc_norm</th>\n",
+       "      <th>sciq/acc</th>\n",
+       "      <th>sciq/acc_norm</th>\n",
+       "      <th>arc/acc</th>\n",
+       "      <th>arc/acc_norm</th>\n",
+       "      <th>mmlu/acc</th>\n",
+       "      <th>mmlu/acc_norm</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>filtering-baseline-2019-18-40gt</td>\n",
+       "      <td>5</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.330953</td>\n",
+       "      <td>0.186</td>\n",
+       "      <td>0.233</td>\n",
+       "      <td>0.272</td>\n",
+       "      <td>0.258</td>\n",
+       "      <td>0.166</td>\n",
+       "      <td>0.286</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.367</td>\n",
+       "      <td>0.362</td>\n",
+       "      <td>0.516</td>\n",
+       "      <td>0.497</td>\n",
+       "      <td>0.210</td>\n",
+       "      <td>0.202</td>\n",
+       "      <td>0.2190</td>\n",
+       "      <td>0.2515</td>\n",
+       "      <td>0.230285</td>\n",
+       "      <td>0.250127</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>filtering-baseline-2019-18-40gt</td>\n",
+       "      <td>5</td>\n",
+       "      <td>1000</td>\n",
+       "      <td>0.357474</td>\n",
+       "      <td>0.239</td>\n",
+       "      <td>0.271</td>\n",
+       "      <td>0.297</td>\n",
+       "      <td>0.287</td>\n",
+       "      <td>0.146</td>\n",
+       "      <td>0.260</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.365</td>\n",
+       "      <td>0.396</td>\n",
+       "      <td>0.503</td>\n",
+       "      <td>0.486</td>\n",
+       "      <td>0.568</td>\n",
+       "      <td>0.502</td>\n",
+       "      <td>0.2665</td>\n",
+       "      <td>0.2855</td>\n",
+       "      <td>0.242526</td>\n",
+       "      <td>0.253291</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>filtering-baseline-2019-18-40gt</td>\n",
+       "      <td>5</td>\n",
+       "      <td>2000</td>\n",
+       "      <td>0.377436</td>\n",
+       "      <td>0.280</td>\n",
+       "      <td>0.284</td>\n",
+       "      <td>0.321</td>\n",
+       "      <td>0.332</td>\n",
+       "      <td>0.134</td>\n",
+       "      <td>0.268</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.368</td>\n",
+       "      <td>0.399</td>\n",
+       "      <td>0.519</td>\n",
+       "      <td>0.502</td>\n",
+       "      <td>0.686</td>\n",
+       "      <td>0.590</td>\n",
+       "      <td>0.3030</td>\n",
+       "      <td>0.3215</td>\n",
+       "      <td>0.245745</td>\n",
+       "      <td>0.260988</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>filtering-baseline-2019-18-40gt</td>\n",
+       "      <td>5</td>\n",
+       "      <td>3000</td>\n",
+       "      <td>0.387994</td>\n",
+       "      <td>0.277</td>\n",
+       "      <td>0.291</td>\n",
+       "      <td>0.339</td>\n",
+       "      <td>0.359</td>\n",
+       "      <td>0.132</td>\n",
+       "      <td>0.280</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.394</td>\n",
+       "      <td>0.404</td>\n",
+       "      <td>0.520</td>\n",
+       "      <td>0.503</td>\n",
+       "      <td>0.721</td>\n",
+       "      <td>0.622</td>\n",
+       "      <td>0.3210</td>\n",
+       "      <td>0.3385</td>\n",
+       "      <td>0.250427</td>\n",
+       "      <td>0.264451</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>filtering-baseline-2019-18-40gt</td>\n",
+       "      <td>5</td>\n",
+       "      <td>4000</td>\n",
+       "      <td>0.396110</td>\n",
+       "      <td>0.299</td>\n",
+       "      <td>0.315</td>\n",
+       "      <td>0.340</td>\n",
+       "      <td>0.366</td>\n",
+       "      <td>0.158</td>\n",
+       "      <td>0.286</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.376</td>\n",
+       "      <td>0.399</td>\n",
+       "      <td>0.515</td>\n",
+       "      <td>0.500</td>\n",
+       "      <td>0.739</td>\n",
+       "      <td>0.620</td>\n",
+       "      <td>0.3320</td>\n",
+       "      <td>0.3445</td>\n",
+       "      <td>0.256134</td>\n",
+       "      <td>0.270382</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>115</th>\n",
+       "      <td>wet-extraction-2019-18</td>\n",
+       "      <td>6</td>\n",
+       "      <td>10000</td>\n",
+       "      <td>0.408977</td>\n",
+       "      <td>0.326</td>\n",
+       "      <td>0.312</td>\n",
+       "      <td>0.362</td>\n",
+       "      <td>0.412</td>\n",
+       "      <td>0.166</td>\n",
+       "      <td>0.312</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.379</td>\n",
+       "      <td>0.396</td>\n",
+       "      <td>0.525</td>\n",
+       "      <td>0.517</td>\n",
+       "      <td>0.767</td>\n",
+       "      <td>0.654</td>\n",
+       "      <td>0.3480</td>\n",
+       "      <td>0.3560</td>\n",
+       "      <td>0.262357</td>\n",
+       "      <td>0.276813</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>116</th>\n",
+       "      <td>wet-extraction-2019-18</td>\n",
+       "      <td>6</td>\n",
+       "      <td>11000</td>\n",
+       "      <td>0.408771</td>\n",
+       "      <td>0.325</td>\n",
+       "      <td>0.315</td>\n",
+       "      <td>0.363</td>\n",
+       "      <td>0.409</td>\n",
+       "      <td>0.162</td>\n",
+       "      <td>0.312</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.388</td>\n",
+       "      <td>0.399</td>\n",
+       "      <td>0.529</td>\n",
+       "      <td>0.520</td>\n",
+       "      <td>0.777</td>\n",
+       "      <td>0.664</td>\n",
+       "      <td>0.3465</td>\n",
+       "      <td>0.3555</td>\n",
+       "      <td>0.261599</td>\n",
+       "      <td>0.276664</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>117</th>\n",
+       "      <td>wet-extraction-2019-18</td>\n",
+       "      <td>6</td>\n",
+       "      <td>12000</td>\n",
+       "      <td>0.408239</td>\n",
+       "      <td>0.329</td>\n",
+       "      <td>0.308</td>\n",
+       "      <td>0.364</td>\n",
+       "      <td>0.416</td>\n",
+       "      <td>0.178</td>\n",
+       "      <td>0.308</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.382</td>\n",
+       "      <td>0.398</td>\n",
+       "      <td>0.521</td>\n",
+       "      <td>0.510</td>\n",
+       "      <td>0.770</td>\n",
+       "      <td>0.656</td>\n",
+       "      <td>0.3555</td>\n",
+       "      <td>0.3595</td>\n",
+       "      <td>0.260928</td>\n",
+       "      <td>0.278411</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>118</th>\n",
+       "      <td>wet-extraction-2019-18</td>\n",
+       "      <td>6</td>\n",
+       "      <td>13000</td>\n",
+       "      <td>0.413263</td>\n",
+       "      <td>0.325</td>\n",
+       "      <td>0.308</td>\n",
+       "      <td>0.367</td>\n",
+       "      <td>0.425</td>\n",
+       "      <td>0.174</td>\n",
+       "      <td>0.312</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.387</td>\n",
+       "      <td>0.411</td>\n",
+       "      <td>0.523</td>\n",
+       "      <td>0.524</td>\n",
+       "      <td>0.774</td>\n",
+       "      <td>0.662</td>\n",
+       "      <td>0.3570</td>\n",
+       "      <td>0.3600</td>\n",
+       "      <td>0.263067</td>\n",
+       "      <td>0.281104</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>119</th>\n",
+       "      <td>wet-extraction-2019-18</td>\n",
+       "      <td>6</td>\n",
+       "      <td>13500</td>\n",
+       "      <td>0.410754</td>\n",
+       "      <td>0.335</td>\n",
+       "      <td>0.310</td>\n",
+       "      <td>0.366</td>\n",
+       "      <td>0.424</td>\n",
+       "      <td>0.164</td>\n",
+       "      <td>0.300</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.392</td>\n",
+       "      <td>0.407</td>\n",
+       "      <td>0.515</td>\n",
+       "      <td>0.519</td>\n",
+       "      <td>0.779</td>\n",
+       "      <td>0.668</td>\n",
+       "      <td>0.3590</td>\n",
+       "      <td>0.3565</td>\n",
+       "      <td>0.261681</td>\n",
+       "      <td>0.279534</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>120 rows × 22 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                             runname  seed  steps  agg_score  \\\n",
+       "0    filtering-baseline-2019-18-40gt     5      0   0.330953   \n",
+       "1    filtering-baseline-2019-18-40gt     5   1000   0.357474   \n",
+       "2    filtering-baseline-2019-18-40gt     5   2000   0.377436   \n",
+       "3    filtering-baseline-2019-18-40gt     5   3000   0.387994   \n",
+       "4    filtering-baseline-2019-18-40gt     5   4000   0.396110   \n",
+       "..                               ...   ...    ...        ...   \n",
+       "115           wet-extraction-2019-18     6  10000   0.408977   \n",
+       "116           wet-extraction-2019-18     6  11000   0.408771   \n",
+       "117           wet-extraction-2019-18     6  12000   0.408239   \n",
+       "118           wet-extraction-2019-18     6  13000   0.413263   \n",
+       "119           wet-extraction-2019-18     6  13500   0.410754   \n",
+       "\n",
+       "     commonsense_qa/acc  commonsense_qa/acc_norm  hellaswag/acc  \\\n",
+       "0                 0.186                    0.233          0.272   \n",
+       "1                 0.239                    0.271          0.297   \n",
+       "2                 0.280                    0.284          0.321   \n",
+       "3                 0.277                    0.291          0.339   \n",
+       "4                 0.299                    0.315          0.340   \n",
+       "..                  ...                      ...            ...   \n",
+       "115               0.326                    0.312          0.362   \n",
+       "116               0.325                    0.315          0.363   \n",
+       "117               0.329                    0.308          0.364   \n",
+       "118               0.325                    0.308          0.367   \n",
+       "119               0.335                    0.310          0.366   \n",
+       "\n",
+       "     hellaswag/acc_norm  openbookqa/acc  openbookqa/acc_norm  ...  siqa/acc  \\\n",
+       "0                 0.258           0.166                0.286  ...     0.367   \n",
+       "1                 0.287           0.146                0.260  ...     0.365   \n",
+       "2                 0.332           0.134                0.268  ...     0.368   \n",
+       "3                 0.359           0.132                0.280  ...     0.394   \n",
+       "4                 0.366           0.158                0.286  ...     0.376   \n",
+       "..                  ...             ...                  ...  ...       ...   \n",
+       "115               0.412           0.166                0.312  ...     0.379   \n",
+       "116               0.409           0.162                0.312  ...     0.388   \n",
+       "117               0.416           0.178                0.308  ...     0.382   \n",
+       "118               0.425           0.174                0.312  ...     0.387   \n",
+       "119               0.424           0.164                0.300  ...     0.392   \n",
+       "\n",
+       "     siqa/acc_norm  winogrande/acc  winogrande/acc_norm  sciq/acc  \\\n",
+       "0            0.362           0.516                0.497     0.210   \n",
+       "1            0.396           0.503                0.486     0.568   \n",
+       "2            0.399           0.519                0.502     0.686   \n",
+       "3            0.404           0.520                0.503     0.721   \n",
+       "4            0.399           0.515                0.500     0.739   \n",
+       "..             ...             ...                  ...       ...   \n",
+       "115          0.396           0.525                0.517     0.767   \n",
+       "116          0.399           0.529                0.520     0.777   \n",
+       "117          0.398           0.521                0.510     0.770   \n",
+       "118          0.411           0.523                0.524     0.774   \n",
+       "119          0.407           0.515                0.519     0.779   \n",
+       "\n",
+       "     sciq/acc_norm  arc/acc  arc/acc_norm  mmlu/acc  mmlu/acc_norm  \n",
+       "0            0.202   0.2190        0.2515  0.230285       0.250127  \n",
+       "1            0.502   0.2665        0.2855  0.242526       0.253291  \n",
+       "2            0.590   0.3030        0.3215  0.245745       0.260988  \n",
+       "3            0.622   0.3210        0.3385  0.250427       0.264451  \n",
+       "4            0.620   0.3320        0.3445  0.256134       0.270382  \n",
+       "..             ...      ...           ...       ...            ...  \n",
+       "115          0.654   0.3480        0.3560  0.262357       0.276813  \n",
+       "116          0.664   0.3465        0.3555  0.261599       0.276664  \n",
+       "117          0.656   0.3555        0.3595  0.260928       0.278411  \n",
+       "118          0.662   0.3570        0.3600  0.263067       0.281104  \n",
+       "119          0.668   0.3590        0.3565  0.261681       0.279534  \n",
+       "\n",
+       "[120 rows x 22 columns]"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "from matplotlib.figure import Figure\n",
+    "\n",
+    "df = pd.read_csv(\"../src_data/wet_comparison.csv\")\n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "b610f43caefdf01",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-13T15:30:52.866635Z",
+     "start_time": "2024-05-13T15:30:52.865068Z"
+    },
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "runs_mapping = {\n",
+    "    \"wet-extraction-2019-18\": \"WET data\",\n",
+    "    \"ind_minhash-CC-MAIN-2019-18\": \"Extracted from WARC\",\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "initial_id",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-13T15:30:53.034617Z",
+     "start_time": "2024-05-13T15:30:52.867342Z"
+    },
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "import os\n",
+    "from matplotlib import pyplot as plt\n",
+    "metrics = ['agg_score', 'commonsense_qa/acc_norm', 'hellaswag/acc_norm', 'openbookqa/acc_norm', 'piqa/acc_norm',\n",
+    "                   'siqa/acc_norm', 'winogrande/acc_norm', 'arc/acc_norm', 'mmlu/acc_norm']\n",
+    "\n",
+    "def normalize_runname(runname):\n",
+    "    return runname.replace(\"/\", \"_\")\n",
+    "\n",
+    "grouped = (\n",
+    "    df.groupby([\"runname\", \"steps\"])\n",
+    "    .agg(\n",
+    "        {\n",
+    "            key: \"mean\" for key in metrics\n",
+    "        }\n",
+    "    )\n",
+    "    .reset_index()\n",
+    ")\n",
+    "\n",
+    "file_id=\"../assets/data/plots/wet_comparison\"\n",
+    "files = {}\n",
+    "for metric in metrics:\n",
+    "    datas = {}\n",
+    "    for name, group in grouped.groupby(\"runname\"):\n",
+    "        if name not in runs_mapping:\n",
+    "            continue\n",
+    "        group = group[[\"steps\", metric]].sort_values(by=\"steps\")\n",
+    "        group = group.set_index(\"steps\")\n",
+    "        rolling_avg = group\n",
+    "        # rolling_avg = group.rolling(window=5).mean()\n",
+    "        datas[name] = {\n",
+    "            \"x\": (rolling_avg.index * 2048 * 1024 * 1e-9).tolist(),\n",
+    "            \"y\": rolling_avg[metric].tolist(),\n",
+    "            \"label\": runs_mapping[name],\n",
+    "        }\n",
+    "    # Sort the datata based on the steps\n",
+    "    datas = {k: v for k, v in sorted(datas.items(), key=lambda x: -x[1][\"y\"][-1])}\n",
+    "    # Create a folder\n",
+    "    os.makedirs(f\"{file_id}\", exist_ok=True)\n",
+    "    with open(f\"{file_id}/{normalize_runname(metric)}.json\", \"w\") as f:\n",
+    "        json.dump({\n",
+    "            \"data\": datas,\n",
+    "            \"layout\": {\n",
+    "                \"title\": {\n",
+    "                    \"text\": \"WET data is worse than data extracted from WARC\"\n",
+    "                },\n",
+    "            }\n",
+    "        }, f)\n",
+    "    files[metric] = {\"file\": f\"{normalize_runname(metric)}.json\"}\n",
+    "# Create index\n",
+    "with open(f\"{file_id}/index.json\", \"w\") as f:\n",
+    "    json.dump({\n",
+    "        \"files\": files,\n",
+    "        \"settings\": {\n",
+    "            \"defaultMetric\": \"agg_score\",\n",
+    "            \"slider\":{\"min\":0,\"max\":10,\"default\":0}\n",
+    "        }\n",
+    "    }, f)\n",
+    "        "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "af28ebbd054cdc33",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-13T15:30:53.036912Z",
+     "start_time": "2024-05-13T15:30:53.035519Z"
+    },
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}