Upload lm-boosted decoder

.gitattributes

@@ -26,3 +26,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zstandard filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 wandb/offline-run-20220203_154548-23cvd7o7/run-23cvd7o7.wandb filter=lfs diff=lfs merge=lfs -text
+5gram.arpa filter=lfs diff=lfs merge=lfs -text
+5gram_correct.arpa filter=lfs diff=lfs merge=lfs -text
+text.txt filter=lfs diff=lfs merge=lfs -text

.ipynb_checkpoints/n-gram-checkpoint.ipynb

@@ -0,0 +1,481 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 51,
+   "id": "831245a1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "2ac8a30f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "target_lang=\"ga-IE\"  # change to your target lang"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 101,
+   "id": "15710167",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Using custom data configuration ga-pl-lang1=ga,lang2=pl\n",
+      "Reusing dataset opus_dgt (/workspace/cache/hf/datasets/opus_dgt/ga-pl-lang1=ga,lang2=pl/0.0.0/a4db75cea3712eb5d4384f0539db82abf897c6b6da5e5e81693e8fd201efc346)\n"
+     ]
+    }
+   ],
+   "source": [
+    "from datasets import load_dataset\n",
+    "\n",
+    "# dataset = load_dataset(\"mozilla-foundation/common_voice_8_0\", \n",
+    "#                        \"ga-IE\", \n",
+    "#                        split=\"train\", \n",
+    "#                        use_auth_token = True)\n",
+    "\n",
+    "dataset = load_dataset(\"opus_dgt\", lang1=\"ga\", lang2=\"pl\", split = 'train')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 102,
+   "id": "fb20d4de",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# ga_txt = [i['ga'] for i in dataset['translation']]\n",
+    "# ga_txt = pd.Series(ga_txt)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 103,
+   "id": "eeca1851",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "chars_to_ignore_regex = '[,?.!\\-\\;\\:\"“%‘”�—’…–]'  # change to the ignored characters of your fine-tuned model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 107,
+   "id": "4df93c9c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import re\n",
+    "\n",
+    "def extract_text(batch):\n",
+    "  text = batch[\"translation\"]\n",
+    "  ga_text = text['ga']\n",
+    "  batch[\"text\"] = re.sub(chars_to_ignore_regex, \"\", ga_text.lower())\n",
+    "  return batch"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 108,
+   "id": "84bedd13",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "d9a11f167bb94faa8e9f6a511407acb4",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "0ex [00:00, ?ex/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "dataset = dataset.map(extract_text, remove_columns=dataset.column_names)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 112,
+   "id": "31cb3c6b",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "342d92a5d9c44c59bcb5dca143ced3b6",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Pushing dataset shards to the dataset hub:   0%|          | 0/1 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "dataset.push_to_hub(f\"{target_lang}_opus_dgt_train\", split=\"train\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "70952673",
+   "metadata": {},
+   "source": [
+    "## N-gram KenLM"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 116,
+   "id": "51756959",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "38d3c229117f4e60a7778f974ac609de",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading:   0%|          | 0.00/1.60k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Using custom data configuration jcmc--ga-IE_opus_dgt_train-aa318da91f5f84f6\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Downloading and preparing dataset opus_dgt/ga-pl (download: 12.11 MiB, generated: 28.99 MiB, post-processed: Unknown size, total: 41.11 MiB) to /workspace/cache/hf/datasets/parquet/jcmc--ga-IE_opus_dgt_train-aa318da91f5f84f6/0.0.0/1638526fd0e8d960534e2155dc54fdff8dce73851f21f031d2fb9c2cf757c121...\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "e5e07f18549b443ead74991a9b338593",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/1 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "0e83c78fa1bc43f19a56b623c92a64a4",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading:   0%|          | 0.00/12.7M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "06649f5cd3324eb49a1bd09b68aa23b6",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/1 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Dataset parquet downloaded and prepared to /workspace/cache/hf/datasets/parquet/jcmc--ga-IE_opus_dgt_train-aa318da91f5f84f6/0.0.0/1638526fd0e8d960534e2155dc54fdff8dce73851f21f031d2fb9c2cf757c121. Subsequent calls will reuse this data.\n"
+     ]
+    }
+   ],
+   "source": [
+    "from datasets import load_dataset\n",
+    "\n",
+    "dataset = load_dataset(\"jcmc/ga-IE_opus_dgt_train\", split=\"train\")\n",
+    "\n",
+    "with open(\"text.txt\", \"w\") as file:\n",
+    "  file.write(\" \".join(dataset[\"text\"]))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 118,
+   "id": "77eb3a41",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=== 1/5 Counting and sorting n-grams ===\n",
+      "Reading /workspace/wav2vec-1b-cv8-ir/text.txt\n",
+      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
+      "****************************************************************************************************\n",
+      "Unigram tokens 4378228 types 70781\n",
+      "=== 2/5 Calculating and sorting adjusted counts ===\n",
+      "Chain sizes: 1:849372 2:14475680768 3:27141902336 4:43427041280 5:63331106816\n",
+      "Statistics:\n",
+      "1 70780 D1=0.684187 D2=1.0538 D3+=1.37643\n",
+      "2 652306 D1=0.766205 D2=1.12085 D3+=1.39031\n",
+      "3 1669326 D1=0.84217 D2=1.20654 D3+=1.39941\n",
+      "4 2514789 D1=0.896214 D2=1.29731 D3+=1.47431\n",
+      "5 3053088 D1=0.794858 D2=1.47897 D3+=1.5117\n",
+      "Memory estimate for binary LM:\n",
+      "type     MB\n",
+      "probing 164 assuming -p 1.5\n",
+      "probing 192 assuming -r models -p 1.5\n",
+      "trie     77 without quantization\n",
+      "trie     42 assuming -q 8 -b 8 quantization \n",
+      "trie     69 assuming -a 22 array pointer compression\n",
+      "trie     34 assuming -a 22 -q 8 -b 8 array pointer compression and quantization\n",
+      "=== 3/5 Calculating and sorting initial probabilities ===\n",
+      "Chain sizes: 1:849360 2:10436896 3:33386520 4:60354936 5:85486464\n",
+      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
+      "####################################################################################################\n",
+      "=== 4/5 Calculating and writing order-interpolated probabilities ===\n",
+      "Chain sizes: 1:849360 2:10436896 3:33386520 4:60354936 5:85486464\n",
+      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
+      "####################################################################################################\n",
+      "=== 5/5 Writing ARPA model ===\n",
+      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
+      "****************************************************************************************************\n",
+      "Name:lmplz\tVmPeak:145097728 kB\tVmRSS:51788 kB\tRSSMax:25679020 kB\tuser:9.15304\tsys:14.1178\tCPU:23.2708\treal:20.9339\n"
+     ]
+    }
+   ],
+   "source": [
+    "!../kenlm/build/bin/lmplz -o 5 <\"text.txt\" > \"5gram.arpa\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 122,
+   "id": "0e043b87",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open(\"5gram.arpa\", \"r\") as read_file, open(\"5gram_correct.arpa\", \"w\") as write_file:\n",
+    "  has_added_eos = False\n",
+    "  for line in read_file:\n",
+    "    if not has_added_eos and \"ngram 1=\" in line:\n",
+    "      count=line.strip().split(\"=\")[-1]\n",
+    "      write_file.write(line.replace(f\"{count}\", f\"{int(count)+1}\"))\n",
+    "    elif not has_added_eos and \"<s>\" in line:\n",
+    "      write_file.write(line)\n",
+    "      write_file.write(line.replace(\"<s>\", \"</s>\"))\n",
+    "      has_added_eos = True\n",
+    "    else:\n",
+    "      write_file.write(line)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 123,
+   "id": "d106c7d1",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\\data\\\n",
+      "ngram 1=70781\n",
+      "ngram 2=652306\n",
+      "ngram 3=1669326\n",
+      "ngram 4=2514789\n",
+      "ngram 5=3053088\n",
+      "\n",
+      "\\1-grams:\n",
+      "-5.8501472\t<unk>\t0\n",
+      "0\t<s>\t-0.11565505\n",
+      "0\t</s>\t-0.11565505\n",
+      "-5.4088216\tmiontuairisc\t-0.20133564\n",
+      "-4.6517477\tcheartaitheach\t-0.24842946\n",
+      "-2.1893916\tmaidir\t-1.7147961\n",
+      "-2.1071756\tle\t-0.7007309\n",
+      "-4.156014\tcoinbhinsiún\t-0.31064242\n",
+      "-1.8876181\tar\t-0.9045828\n",
+      "-4.62287\tdhlínse\t-0.24268326\n",
+      "-1.6051095\tagus\t-0.8729715\n",
+      "-4.1465816\taithint\t-0.21693327\n"
+     ]
+    }
+   ],
+   "source": [
+    "!head -20 5gram_correct.arpa"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 124,
+   "id": "85ef4c43",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import AutoProcessor\n",
+    "\n",
+    "processor = AutoProcessor.from_pretrained(\"./\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 125,
+   "id": "cb2a2768",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "vocab_dict = processor.tokenizer.get_vocab()\n",
+    "sorted_vocab_dict = {k.lower(): v for k, v in sorted(vocab_dict.items(), key=lambda item: item[1])}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 126,
+   "id": "d19eee6f",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Found entries of length > 1 in alphabet. This is unusual unless style is BPE, but the alphabet was not recognized as BPE type. Is this correct?\n",
+      "Unigrams and labels don't seem to agree.\n"
+     ]
+    }
+   ],
+   "source": [
+    "from pyctcdecode import build_ctcdecoder\n",
+    "\n",
+    "decoder = build_ctcdecoder(\n",
+    "    labels=list(sorted_vocab_dict.keys()),\n",
+    "    kenlm_model_path=\"5gram_correct.arpa\",\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 127,
+   "id": "4e8031a9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import Wav2Vec2ProcessorWithLM\n",
+    "\n",
+    "processor_with_lm = Wav2Vec2ProcessorWithLM(\n",
+    "    feature_extractor=processor.feature_extractor,\n",
+    "    tokenizer=processor.tokenizer,\n",
+    "    decoder=decoder\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 128,
+   "id": "6f32faf4",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/workspace/wav2vec-1b-cv8-ir/./ is already a clone of https://huggingface.co/jcmc/wav2vec-1b-cv8-ir. Make sure you pull the latest changes with `repo.git_pull()`.\n"
+     ]
+    }
+   ],
+   "source": [
+    "from huggingface_hub import Repository\n",
+    "\n",
+    "repo = Repository(local_dir=\"./\", clone_from=\"jcmc/wav2vec-1b-cv8-ir\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 129,
+   "id": "a7e91068",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'/workspace/wav2vec-1b-cv8-ir'"
+      ]
+     },
+     "execution_count": 129,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "pwd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0a1de336",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "processor_with_lm.save_pretrained(\"xls-r-300m-sv\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

.ipynb_checkpoints/preprocessor_config-checkpoint.json

@@ -0,0 +1,9 @@
+{
+  "do_normalize": true,
+  "feature_extractor_type": "Wav2Vec2FeatureExtractor",
+  "feature_size": 1,
+  "padding_side": "right",
+  "padding_value": 0,
+  "return_attention_mask": true,
+  "sampling_rate": 16000
+}

.ipynb_checkpoints/tokenizer_config-checkpoint.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "special_tokens_map_file": null, "tokenizer_file": null, "name_or_path": "./", "tokenizer_class": "Wav2Vec2CTCTokenizer", "processor_class": "Wav2Vec2ProcessorWithLM"}

5gram.arpa

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07a5b3058d8cca7e1a61aa31b7ab0907fdb6ff7a104dfef12d8d470b2513c391
+size 376008972

5gram_correct.arpa

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8d79210ff27e6e122fa9af6411f860d85ca20ecac3d76bb4d716341b467e7a8
+size 376008991

alphabet.json

@@ -0,0 +1 @@
+{"labels": [" ", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "r", "s", "t", "u", "v", "w", "x", "y", "\u00e1", "\u00e9", "\u00ed", "\u00f3", "\u00fa", "\u2047", "", "<s>", "</s>"], "is_bpe": false}

language_model/5gram.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1238bb0e9f91c4250009bb8496c4732cad0da6f6a9fbaa945cb5782af4a4bbdc
+size 173705975

language_model/attrs.json

@@ -0,0 +1 @@
+{"alpha": 0.5, "beta": 1.5, "unk_score_offset": -10.0, "score_boundary": true}

n-gram.ipynb

@@ -0,0 +1,450 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "e960dfd7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "7168a253",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "target_lang=\"ga-IE\"  # change to your target lang"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 101,
+   "id": "e170befe",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Using custom data configuration ga-pl-lang1=ga,lang2=pl\n",
+      "Reusing dataset opus_dgt (/workspace/cache/hf/datasets/opus_dgt/ga-pl-lang1=ga,lang2=pl/0.0.0/a4db75cea3712eb5d4384f0539db82abf897c6b6da5e5e81693e8fd201efc346)\n"
+     ]
+    }
+   ],
+   "source": [
+    "from datasets import load_dataset\n",
+    "\n",
+    "# dataset = load_dataset(\"mozilla-foundation/common_voice_8_0\", \n",
+    "#                        \"ga-IE\", \n",
+    "#                        split=\"train\", \n",
+    "#                        use_auth_token = True)\n",
+    "\n",
+    "# dataset = load_dataset(\"opus_dgt\", lang1=\"ga\", lang2=\"pl\", split = 'train')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "33973bd4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# ga_txt = [i['ga'] for i in dataset['translation']]\n",
+    "# ga_txt = pd.Series(ga_txt)\n",
+    "\n",
+    "chars_to_ignore_regex = '[,?.!\\-\\;\\:\"“%‘”�—’…–]'  # change to the ignored characters of your fine-tuned model\n",
+    "\n",
+    "import re\n",
+    "\n",
+    "def extract_text(batch):\n",
+    "  text = batch[\"translation\"]\n",
+    "  ga_text = text['ga']\n",
+    "  batch[\"text\"] = re.sub(chars_to_ignore_regex, \"\", ga_text.lower())\n",
+    "  return batch\n",
+    "\n",
+    "# dataset = dataset.map(extract_text, remove_columns=dataset.column_names)\n",
+    "\n",
+    "# dataset.push_to_hub(f\"{target_lang}_opus_dgt_train\", split=\"train\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "53e62728",
+   "metadata": {},
+   "source": [
+    "## N-gram KenLM"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "cb04cc9d",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "0c3dbd6368014788bff9249dd460d03e",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading:   0%|          | 0.00/1.60k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Using custom data configuration jcmc--ga-IE_opus_dgt_train-aa318da91f5f84f6\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Downloading and preparing dataset opus_dgt/ga-pl (download: 12.11 MiB, generated: 28.99 MiB, post-processed: Unknown size, total: 41.11 MiB) to /workspace/cache/hf/datasets/parquet/jcmc--ga-IE_opus_dgt_train-aa318da91f5f84f6/0.0.0/1638526fd0e8d960534e2155dc54fdff8dce73851f21f031d2fb9c2cf757c121...\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "42c92d51527a41fd91a38c13265c4ea6",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/1 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "ae0badc4154f4fc586d3fc415d70c06a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading:   0%|          | 0.00/12.7M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "f25b9f17355149df880331f926c76279",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/1 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Dataset parquet downloaded and prepared to /workspace/cache/hf/datasets/parquet/jcmc--ga-IE_opus_dgt_train-aa318da91f5f84f6/0.0.0/1638526fd0e8d960534e2155dc54fdff8dce73851f21f031d2fb9c2cf757c121. Subsequent calls will reuse this data.\n"
+     ]
+    }
+   ],
+   "source": [
+    "from datasets import load_dataset\n",
+    "\n",
+    "dataset = load_dataset(\"jcmc/ga-IE_opus_dgt_train\", split=\"train\")\n",
+    "\n",
+    "with open(\"text.txt\", \"w\") as file:\n",
+    "  file.write(\" \".join(dataset[\"text\"]))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "06ce00d3",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=== 1/5 Counting and sorting n-grams ===\n",
+      "Reading /workspace/wav2vec-cv7-1b-ir/text.txt\n",
+      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
+      "****************************************************************************************************\n",
+      "Unigram tokens 4378228 types 70781\n",
+      "=== 2/5 Calculating and sorting adjusted counts ===\n",
+      "Chain sizes: 1:849372 2:14476327936 3:27143116800 4:43428982784 5:63333937152\n",
+      "Statistics:\n",
+      "1 70780 D1=0.684187 D2=1.0538 D3+=1.37643\n",
+      "2 652306 D1=0.766205 D2=1.12085 D3+=1.39031\n",
+      "3 1669326 D1=0.84217 D2=1.20654 D3+=1.39941\n",
+      "4 2514789 D1=0.896214 D2=1.29731 D3+=1.47431\n",
+      "5 3053088 D1=0.794858 D2=1.47897 D3+=1.5117\n",
+      "Memory estimate for binary LM:\n",
+      "type     MB\n",
+      "probing 164 assuming -p 1.5\n",
+      "probing 192 assuming -r models -p 1.5\n",
+      "trie     77 without quantization\n",
+      "trie     42 assuming -q 8 -b 8 quantization \n",
+      "trie     69 assuming -a 22 array pointer compression\n",
+      "trie     34 assuming -a 22 -q 8 -b 8 array pointer compression and quantization\n",
+      "=== 3/5 Calculating and sorting initial probabilities ===\n",
+      "Chain sizes: 1:849360 2:10436896 3:33386520 4:60354936 5:85486464\n",
+      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
+      "####################################################################################################\n",
+      "=== 4/5 Calculating and writing order-interpolated probabilities ===\n",
+      "Chain sizes: 1:849360 2:10436896 3:33386520 4:60354936 5:85486464\n",
+      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
+      "####################################################################################################\n",
+      "=== 5/5 Writing ARPA model ===\n",
+      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
+      "****************************************************************************************************\n",
+      "Name:lmplz\tVmPeak:145104204 kB\tVmRSS:51852 kB\tRSSMax:25679996 kB\tuser:9.46174\tsys:23.4312\tCPU:32.893\treal:30.3848\n"
+     ]
+    }
+   ],
+   "source": [
+    "!../kenlm/build/bin/lmplz -o 5 <\"text.txt\" > \"5gram.arpa\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "e076416d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open(\"5gram.arpa\", \"r\") as read_file, open(\"5gram_correct.arpa\", \"w\") as write_file:\n",
+    "  has_added_eos = False\n",
+    "  for line in read_file:\n",
+    "    if not has_added_eos and \"ngram 1=\" in line:\n",
+    "      count=line.strip().split(\"=\")[-1]\n",
+    "      write_file.write(line.replace(f\"{count}\", f\"{int(count)+1}\"))\n",
+    "    elif not has_added_eos and \"<s>\" in line:\n",
+    "      write_file.write(line)\n",
+    "      write_file.write(line.replace(\"<s>\", \"</s>\"))\n",
+    "      has_added_eos = True\n",
+    "    else:\n",
+    "      write_file.write(line)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "34ac1708",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\\data\\\n",
+      "ngram 1=70781\n",
+      "ngram 2=652306\n",
+      "ngram 3=1669326\n",
+      "ngram 4=2514789\n",
+      "ngram 5=3053088\n",
+      "\n",
+      "\\1-grams:\n",
+      "-5.8501472\t<unk>\t0\n",
+      "0\t<s>\t-0.11565505\n",
+      "0\t</s>\t-0.11565505\n",
+      "-5.4088216\tmiontuairisc\t-0.20133564\n",
+      "-4.6517477\tcheartaitheach\t-0.24842946\n",
+      "-2.1893916\tmaidir\t-1.7147961\n",
+      "-2.1071756\tle\t-0.7007309\n",
+      "-4.156014\tcoinbhinsiún\t-0.31064242\n",
+      "-1.8876181\tar\t-0.9045828\n",
+      "-4.62287\tdhlínse\t-0.24268326\n",
+      "-1.6051095\tagus\t-0.8729715\n",
+      "-4.1465816\taithint\t-0.21693327\n"
+     ]
+    }
+   ],
+   "source": [
+    "!head -20 5gram_correct.arpa"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "a096b154",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import AutoProcessor\n",
+    "\n",
+    "processor = AutoProcessor.from_pretrained(\"./\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "097ae051",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "vocab_dict = processor.tokenizer.get_vocab()\n",
+    "sorted_vocab_dict = {k.lower(): v for k, v in sorted(vocab_dict.items(), key=lambda item: item[1])}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "edeb35c3",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Found entries of length > 1 in alphabet. This is unusual unless style is BPE, but the alphabet was not recognized as BPE type. Is this correct?\n",
+      "Unigrams and labels don't seem to agree.\n"
+     ]
+    }
+   ],
+   "source": [
+    "from pyctcdecode import build_ctcdecoder\n",
+    "\n",
+    "decoder = build_ctcdecoder(\n",
+    "    labels=list(sorted_vocab_dict.keys()),\n",
+    "    kenlm_model_path=\"5gram_correct.arpa\",\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "3e8debd2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import Wav2Vec2ProcessorWithLM\n",
+    "\n",
+    "processor_with_lm = Wav2Vec2ProcessorWithLM(\n",
+    "    feature_extractor=processor.feature_extractor,\n",
+    "    tokenizer=processor.tokenizer,\n",
+    "    decoder=decoder\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "e8f3f674",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/workspace/wav2vec-cv7-1b-ir/./ is already a clone of https://huggingface.co/jcmc/wav2vec-cv7-1b-ir. Make sure you pull the latest changes with `repo.git_pull()`.\n"
+     ]
+    }
+   ],
+   "source": [
+    "from huggingface_hub import Repository\n",
+    "\n",
+    "repo = Repository(local_dir=\"./\", clone_from=\"jcmc/wav2vec-cv7-1b-ir\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "a260b7f2",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'/workspace/wav2vec-cv7-1b-ir'"
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "pwd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "b5958d5e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "processor_with_lm.save_pretrained(\"./\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "34798422",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Reading ./language_model/5gram_correct.arpa\n",
+      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
+      "****************************************************************************************************\n",
+      "SUCCESS\n"
+     ]
+    }
+   ],
+   "source": [
+    "!../kenlm/build/bin/build_binary ./language_model/5gram_correct.arpa ./language_model/5gram.bin"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8f2900a8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "repo.push_to_hub(commit_message=\"Upload lm-boosted decoder\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

preprocessor_config.json

@@ -4,6 +4,7 @@
   "feature_size": 1,
   "padding_side": "right",
   "padding_value": 0,
+  "processor_class": "Wav2Vec2ProcessorWithLM",
   "return_attention_mask": true,
   "sampling_rate": 16000
 }

special_tokens_map.json

@@ -1 +1 @@
-{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}

text.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab790db751ff5b9ddc7575cb8fe62d57c8a4b8fdda4950b2f9a17aad421bfffc
+size 29862507

tokenizer_config.json

@@ -1 +1 @@
-{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "special_tokens_map_file": null, "tokenizer_file": null, "name_or_path": "./", "tokenizer_class": "Wav2Vec2CTCTokenizer"}
+{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "special_tokens_map_file": null, "tokenizer_file": null, "name_or_path": "./", "tokenizer_class": "Wav2Vec2CTCTokenizer", "processor_class": "Wav2Vec2ProcessorWithLM"}

wandb/offline-run-20220203_154548-23cvd7o7/files/output.log

@@ -0,0 +1,144 @@
+{'loss': 10.7305, 'learning_rate': 1.9200000000000003e-06, 'epoch': 3.12}
+{'loss': 3.0098, 'learning_rate': 3.920000000000001e-06, 'epoch': 6.25}
+{'loss': 2.9327, 'learning_rate': 5.92e-06, 'epoch': 9.37}
+{'loss': 2.8216, 'learning_rate': 7.92e-06, 'epoch': 12.49}
+{'loss': 2.3731, 'learning_rate': 9.920000000000002e-06, 'epoch': 15.62}
+{'eval_loss': 1.5517226457595825, 'eval_wer': 0.9499121265377856, 'eval_runtime': 24.8327, 'eval_samples_per_second': 20.497, 'eval_steps_per_second': 1.289, 'epoch': 15.62}
+{'loss': 1.9105, 'learning_rate': 1.1920000000000001e-05, 'epoch': 18.74}
+{'loss': 1.714, 'learning_rate': 1.392e-05, 'epoch': 21.86}
+{'loss': 1.5476, 'learning_rate': 1.5920000000000003e-05, 'epoch': 24.98}
+{'loss': 1.4238, 'learning_rate': 1.792e-05, 'epoch': 28.12}
+{'loss': 1.3312, 'learning_rate': 1.9920000000000002e-05, 'epoch': 31.25}
+{'eval_loss': 0.8717297911643982, 'eval_wer': 0.6189220855301699, 'eval_runtime': 24.7966, 'eval_samples_per_second': 20.527, 'eval_steps_per_second': 1.29, 'epoch': 31.25}
+{'loss': 1.2049, 'learning_rate': 1.912727272727273e-05, 'epoch': 34.37}
+{'loss': 1.1346, 'learning_rate': 1.821818181818182e-05, 'epoch': 37.49}
+{'loss': 1.0533, 'learning_rate': 1.730909090909091e-05, 'epoch': 40.62}
+{'loss': 0.9638, 'learning_rate': 1.64e-05, 'epoch': 43.74}
+{'loss': 0.9135, 'learning_rate': 1.549090909090909e-05, 'epoch': 46.86}
+{'eval_loss': 0.8298946619033813, 'eval_wer': 0.5310486233157586, 'eval_runtime': 24.721, 'eval_samples_per_second': 20.59, 'eval_steps_per_second': 1.294, 'epoch': 46.86}
+{'loss': 0.8568, 'learning_rate': 1.4581818181818184e-05, 'epoch': 49.98}
+{'loss': 0.8141, 'learning_rate': 1.3672727272727273e-05, 'epoch': 53.12}
+{'loss': 0.7526, 'learning_rate': 1.2763636363636365e-05, 'epoch': 56.25}
+{'loss': 0.7177, 'learning_rate': 1.1854545454545457e-05, 'epoch': 59.37}
+{'loss': 0.6719, 'learning_rate': 1.0945454545454545e-05, 'epoch': 62.49}
+{'eval_loss': 0.8842366933822632, 'eval_wer': 0.5043936731107206, 'eval_runtime': 25.0435, 'eval_samples_per_second': 20.325, 'eval_steps_per_second': 1.278, 'epoch': 62.49}
+{'loss': 0.6552, 'learning_rate': 1.0036363636363637e-05, 'epoch': 65.62}
+{'loss': 0.6145, 'learning_rate': 9.127272727272727e-06, 'epoch': 68.74}
+{'loss': 0.596, 'learning_rate': 8.21818181818182e-06, 'epoch': 71.86}
+{'loss': 0.5719, 'learning_rate': 7.30909090909091e-06, 'epoch': 74.98}
+{'loss': 0.5583, 'learning_rate': 6.4000000000000006e-06, 'epoch': 78.12}
+{'eval_loss': 0.9093144536018372, 'eval_wer': 0.4800820152314001, 'eval_runtime': 24.6074, 'eval_samples_per_second': 20.685, 'eval_steps_per_second': 1.3, 'epoch': 78.12}
+{'loss': 0.5417, 'learning_rate': 5.490909090909091e-06, 'epoch': 81.25}
+{'loss': 0.5241, 'learning_rate': 4.581818181818183e-06, 'epoch': 84.37}
+{'loss': 0.4901, 'learning_rate': 3.672727272727273e-06, 'epoch': 87.49}
+{'loss': 0.4882, 'learning_rate': 2.763636363636364e-06, 'epoch': 90.62}
+{'loss': 0.4728, 'learning_rate': 1.8545454545454546e-06, 'epoch': 93.74}
+{'eval_loss': 0.9488239884376526, 'eval_wer': 0.48125366139425896, 'eval_runtime': 24.6884, 'eval_samples_per_second': 20.617, 'eval_steps_per_second': 1.296, 'epoch': 93.74}
+{'loss': 0.4682, 'learning_rate': 9.454545454545455e-07, 'epoch': 96.86}
+{'loss': 0.4634, 'learning_rate': 3.636363636363637e-08, 'epoch': 99.98}
+{'train_runtime': 8387.3816, 'train_samples_per_second': 12.34, 'train_steps_per_second': 0.382, 'train_loss': 1.4163260304927825, 'epoch': 99.98}
+***** train metrics *****
+  epoch                    =      99.98
+  train_loss               =     1.4163
+  train_runtime            = 2:19:47.38
+  train_samples            =       1035
+  train_samples_per_second =      12.34
+  train_steps_per_second   =      0.382
+02/03/2022 18:01:03 - INFO - __main__ - *** Evaluate ***
+***** eval metrics *****
+  epoch                   =      99.98
+  eval_loss               =     0.9562
+  eval_runtime            = 0:00:24.83
+  eval_samples            =        509
+  eval_samples_per_second =     20.497
+  eval_steps_per_second   =      1.289
+  eval_wer                =     0.4801
+02/03/2022 18:04:19 - WARNING - huggingface_hub.repository - Adding files tracked by Git LFS: ['wandb/offline-run-20220203_154548-23cvd7o7/run-23cvd7o7.wandb']. This may take a bit of time if the files are large.
+02/03/2022 18:05:13 - WARNING - huggingface_hub.repository - Several commits (2) will be pushed upstream.
+02/03/2022 18:05:13 - WARNING - huggingface_hub.repository - The progress bars may be unreliable.
+02/03/2022 18:07:27 - WARNING - huggingface_hub.repository - To https://huggingface.co/jcmc/wav2vec-cv7-1b-ir
+   f30c4d7..a0c1812  main -> main
+02/03/2022 18:07:33 - WARNING - huggingface_hub.repository - To https://huggingface.co/jcmc/wav2vec-cv7-1b-ir
+   a0c1812..e90ef2f  main -> main
+ 16%|█████████████████████████                                                                                                                                       | 500/3200 [20:22<1:57:09,  2.60s/it]The following columns in the evaluation set  don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.
+***** Running Evaluation *****
+  Num examples = 509
+  Batch size = 16
+ 16%|█████████████████████████                                                                                                                                       | 500/3200 [20:47<1:57:09,  2.60s/it]Saving model checkpoint to ./checkpoint-500
+Configuration saved in ./checkpoint-500/config.json
+Model weights saved in ./checkpoint-500/pytorch_model.bin
+Configuration saved in ./checkpoint-500/preprocessor_config.json
+Configuration saved in ./preprocessor_config.json
+ 31%|█████████████████████████████████████████████████▋                                                                                                             | 1000/3200 [42:27<1:10:05,  1.91s/it]The following columns in the evaluation set  don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.
+***** Running Evaluation *****
+  Num examples = 509
+  Batch size = 16
+ 31%|█████████████████████████████████████████████████▋                                                                                                             | 1000/3200 [42:51<1:10:05,  1.91s/it]Saving model checkpoint to ./checkpoint-1000
+Configuration saved in ./checkpoint-1000/config.json
+Model weights saved in ./checkpoint-1000/pytorch_model.bin
+Configuration saved in ./checkpoint-1000/preprocessor_config.json
+ 47%|█████████████████████████████████████████████████████████████████████████▌                                                                                   | 1500/3200 [1:03:33<1:10:34,  2.49s/it]The following columns in the evaluation set  don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.
+***** Running Evaluation *****
+  Num examples = 509
+  Batch size = 16
+ 47%|█████████████████████████████████████████████████████████████████████████▌                                                                                   | 1500/3200 [1:03:57<1:10:34,  2.49s/it]Saving model checkpoint to ./checkpoint-1500
+Configuration saved in ./checkpoint-1500/config.json
+Model weights saved in ./checkpoint-1500/pytorch_model.bin
+Configuration saved in ./checkpoint-1500/preprocessor_config.json
+ 62%|███████████████████████████████████████████████████████████████████████████████████████████████████▍                                                           | 2000/3200 [1:24:29<36:01,  1.80s/it]The following columns in the evaluation set  don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.
+***** Running Evaluation *****
+  Num examples = 509
+  Batch size = 16
+ 62%|███████████████████████████████████████████████████████████████████████████████████████████████████▍                                                           | 2000/3200 [1:24:54<36:01,  1.80s/it]Saving model checkpoint to ./checkpoint-2000
+Configuration saved in ./checkpoint-2000/config.json
+Model weights saved in ./checkpoint-2000/pytorch_model.bin
+Configuration saved in ./checkpoint-2000/preprocessor_config.json
+Deleting older checkpoint [checkpoint-500] due to args.save_total_limit
+ 78%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏                                  | 2500/3200 [1:45:29<31:58,  2.74s/it]The following columns in the evaluation set  don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.
+***** Running Evaluation *****
+  Num examples = 509
+  Batch size = 16
+ 78%|████████████████████████████████████████████████████████��███████████████████████████████████████████████████████████████████▏                                  | 2500/3200 [1:45:54<31:58,  2.74s/it]Saving model checkpoint to ./checkpoint-2500
+Configuration saved in ./checkpoint-2500/config.json
+Model weights saved in ./checkpoint-2500/pytorch_model.bin
+Configuration saved in ./checkpoint-2500/preprocessor_config.json
+Deleting older checkpoint [checkpoint-1000] due to args.save_total_limit
+ 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████          | 3000/3200 [2:06:18<05:55,  1.78s/it]The following columns in the evaluation set  don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.
+***** Running Evaluation *****
+  Num examples = 509
+  Batch size = 16
+ 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████          | 3000/3200 [2:06:42<05:55,  1.78s/it]Saving model checkpoint to ./checkpoint-3000
+Configuration saved in ./checkpoint-3000/config.json
+Model weights saved in ./checkpoint-3000/pytorch_model.bin
+Configuration saved in ./checkpoint-3000/preprocessor_config.json
+Deleting older checkpoint [checkpoint-1500] due to args.save_total_limit
+100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3200/3200 [2:15:04<00:00,  1.84s/it]
+Training completed. Do not forget to share your model on huggingface.co/models =)
+100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3200/3200 [2:15:04<00:00,  2.53s/it]
+Saving model checkpoint to ./
+Configuration saved in ./config.json
+Model weights saved in ./pytorch_model.bin
+Configuration saved in ./preprocessor_config.json
+The following columns in the evaluation set  don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.
+***** Running Evaluation *****
+  Num examples = 509
+  Batch size = 16
+100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:24<00:00,  1.33it/s]
+Saving model checkpoint to ./
+Configuration saved in ./config.json
+Model weights saved in ./pytorch_model.bin
+Configuration saved in ./preprocessor_config.json
+Adding files tracked by Git LFS: ['wandb/offline-run-20220203_154548-23cvd7o7/run-23cvd7o7.wandb']. This may take a bit of time if the files are large.
+Several commits (2) will be pushed upstream.
+The progress bars may be unreliable.
+Upload file pytorch_model.bin: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋| 3.58G/3.59G [02:11<00:00, 30.5MB/s]To https://huggingface.co/jcmc/wav2vec-cv7-1b-ir
+   f30c4d7..a0c1812  main -> main20203_154548-23cvd7o7/run-23cvd7o7.wandb: 100%|█████████████████████████████████████████████████████████████████████████████████████| 39.6M/39.6M [00:18<00:00, 19.7MB/s]
+Upload file runs/Feb03_15-40-29_job-829e2c87-5501-41ef-ad65-a05e9b64bfd7/events.out.tfevents.1643911287.job-829e2c87-5501-41ef-ad65-a05e9b64bfd7.27059.2: 100%|██████████████████| 358/358 [00:00<?, ?B/s]
+Upload file pytorch_model.bin: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3.59G/3.59G [02:12<00:00, 29.1MB/s]
+Upload file wandb/offline-run-20220203_154548-23cvd7o7/run-23cvd7o7.wandb: 100%|██████████████████████████████████████████████████████████████████████████████████████| 39.6M/39.6M [02:12<00:00, 314kB/s]
+Upload file runs/Feb03_15-40-29_job-829e2c87-5501-41ef-ad65-a05e9b64bfd7/events.out.tfevents.1643911287.job-829e2c87-5501-41ef-ad65-a05e9b64bfd7.27059.2: 100%|██████████████████| 358/358 [02:12<?, ?B/s]
+Upload file runs/Feb03_15-40-29_job-829e2c87-5501-41ef-ad65-a05e9b64bfd7/events.out.tfevents.1643902867.job-829e2c87-5501-41ef-ad65-a05e9b64bfd7.27059.0: 100%|███████| 11.7k/11.7k [02:12<00:00, 64.2B/s]
+Dropping the following result as it does not have all the necessary fields:ents.out.tfevents.1643911287.job-829e2c87-5501-41ef-ad65-a05e9b64bfd7.27059.2: 100%|██████████████████| 358/358 [02:12<?, ?B/s]
+{'dataset': {'name': 'MOZILLA-FOUNDATION/COMMON_VOICE_7_0 - GA-IE', 'type': 'common_voice', 'args': 'Config: ga-IE, Training split: train+validation, Eval split: test'}}
+To https://huggingface.co/jcmc/wav2vec-cv7-1b-ir1-41ef-ad65-a05e9b64bfd7/events.out.tfevents.1643902867.job-829e2c87-5501-41ef-ad65-a05e9b64bfd7.27059.0: 100%|███████| 11.7k/11.7k [02:12<00:00, 64.2B/s]
+   a0c1812..e90ef2f  main -> main

wandb/offline-run-20220203_154548-23cvd7o7/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb": {"runtime": 8510}}

wandb/offline-run-20220203_154548-23cvd7o7/logs/debug-internal.log

@@ -11,3 +11,71 @@
 2022-02-03 15:45:49,618 DEBUG   HandlerThread:29156 [meta.py:_save_conda():79] save conda
 2022-02-03 15:45:50,243 DEBUG   HandlerThread:29156 [meta.py:_save_conda():89] save conda done
 2022-02-03 15:45:50,243 DEBUG   HandlerThread:29156 [meta.py:probe():252] probe done
+2022-02-03 18:07:40,029 DEBUG   SenderThread:29156 [sender.py:send():234] send: exit
+2022-02-03 18:07:40,030 DEBUG   HandlerThread:29156 [handler.py:handle_request():130] handle_request: poll_exit
+2022-02-03 18:07:40,030 INFO    SenderThread:29156 [sender.py:send_exit():366] handling exit code: 0
+2022-02-03 18:07:40,031 INFO    SenderThread:29156 [sender.py:send_exit():368] handling runtime: 8510
+2022-02-03 18:07:40,032 INFO    SenderThread:29156 [sender.py:_save_file():939] saving file wandb-summary.json with policy end
+2022-02-03 18:07:40,032 INFO    SenderThread:29156 [sender.py:send_exit():374] send defer
+2022-02-03 18:07:40,032 DEBUG   SenderThread:29156 [sender.py:send_request():248] send_request: poll_exit
+2022-02-03 18:07:40,033 DEBUG   HandlerThread:29156 [handler.py:handle_request():130] handle_request: defer
+2022-02-03 18:07:40,033 INFO    HandlerThread:29156 [handler.py:handle_request_defer():147] handle defer: 0
+2022-02-03 18:07:40,034 DEBUG   SenderThread:29156 [sender.py:send_request():248] send_request: defer
+2022-02-03 18:07:40,034 INFO    SenderThread:29156 [sender.py:send_request_defer():383] handle sender defer: 0
+2022-02-03 18:07:40,034 INFO    SenderThread:29156 [sender.py:transition_state():387] send defer: 1
+2022-02-03 18:07:40,035 DEBUG   HandlerThread:29156 [handler.py:handle_request():130] handle_request: defer
+2022-02-03 18:07:40,035 INFO    HandlerThread:29156 [handler.py:handle_request_defer():147] handle defer: 1
+2022-02-03 18:07:40,106 DEBUG   SenderThread:29156 [sender.py:send_request():248] send_request: defer
+2022-02-03 18:07:40,107 INFO    SenderThread:29156 [sender.py:send_request_defer():383] handle sender defer: 1
+2022-02-03 18:07:40,107 INFO    SenderThread:29156 [sender.py:transition_state():387] send defer: 2
+2022-02-03 18:07:40,108 DEBUG   HandlerThread:29156 [handler.py:handle_request():130] handle_request: defer
+2022-02-03 18:07:40,108 INFO    HandlerThread:29156 [handler.py:handle_request_defer():147] handle defer: 2
+2022-02-03 18:07:40,108 DEBUG   SenderThread:29156 [sender.py:send_request():248] send_request: defer
+2022-02-03 18:07:40,108 INFO    SenderThread:29156 [sender.py:send_request_defer():383] handle sender defer: 2
+2022-02-03 18:07:40,108 INFO    SenderThread:29156 [sender.py:transition_state():387] send defer: 3
+2022-02-03 18:07:40,109 DEBUG   HandlerThread:29156 [handler.py:handle_request():130] handle_request: defer
+2022-02-03 18:07:40,109 INFO    HandlerThread:29156 [handler.py:handle_request_defer():147] handle defer: 3
+2022-02-03 18:07:40,164 DEBUG   HandlerThread:29156 [handler.py:handle_request():130] handle_request: poll_exit
+2022-02-03 18:07:40,168 DEBUG   SenderThread:29156 [sender.py:send_request():248] send_request: defer
+2022-02-03 18:07:40,168 INFO    SenderThread:29156 [sender.py:send_request_defer():383] handle sender defer: 3
+2022-02-03 18:07:40,168 INFO    SenderThread:29156 [sender.py:transition_state():387] send defer: 4
+2022-02-03 18:07:40,169 DEBUG   SenderThread:29156 [sender.py:send_request():248] send_request: poll_exit
+2022-02-03 18:07:40,169 DEBUG   HandlerThread:29156 [handler.py:handle_request():130] handle_request: defer
+2022-02-03 18:07:40,170 INFO    HandlerThread:29156 [handler.py:handle_request_defer():147] handle defer: 4
+2022-02-03 18:07:40,170 DEBUG   SenderThread:29156 [sender.py:send_request():248] send_request: defer
+2022-02-03 18:07:40,170 INFO    SenderThread:29156 [sender.py:send_request_defer():383] handle sender defer: 4
+2022-02-03 18:07:40,170 INFO    SenderThread:29156 [sender.py:transition_state():387] send defer: 5
+2022-02-03 18:07:40,171 DEBUG   HandlerThread:29156 [handler.py:handle_request():130] handle_request: defer
+2022-02-03 18:07:40,171 INFO    HandlerThread:29156 [handler.py:handle_request_defer():147] handle defer: 5
+2022-02-03 18:07:40,171 DEBUG   SenderThread:29156 [sender.py:send_request():248] send_request: defer
+2022-02-03 18:07:40,171 INFO    SenderThread:29156 [sender.py:send_request_defer():383] handle sender defer: 5
+2022-02-03 18:07:40,172 INFO    SenderThread:29156 [sender.py:transition_state():387] send defer: 6
+2022-02-03 18:07:40,172 DEBUG   HandlerThread:29156 [handler.py:handle_request():130] handle_request: defer
+2022-02-03 18:07:40,172 INFO    HandlerThread:29156 [handler.py:handle_request_defer():147] handle defer: 6
+2022-02-03 18:07:40,172 DEBUG   SenderThread:29156 [sender.py:send_request():248] send_request: defer
+2022-02-03 18:07:40,173 INFO    SenderThread:29156 [sender.py:send_request_defer():383] handle sender defer: 6
+2022-02-03 18:07:40,173 INFO    SenderThread:29156 [sender.py:transition_state():387] send defer: 7
+2022-02-03 18:07:40,173 DEBUG   HandlerThread:29156 [handler.py:handle_request():130] handle_request: defer
+2022-02-03 18:07:40,173 INFO    HandlerThread:29156 [handler.py:handle_request_defer():147] handle defer: 7
+2022-02-03 18:07:40,173 DEBUG   SenderThread:29156 [sender.py:send_request():248] send_request: defer
+2022-02-03 18:07:40,174 INFO    SenderThread:29156 [sender.py:send_request_defer():383] handle sender defer: 7
+2022-02-03 18:07:40,174 INFO    SenderThread:29156 [sender.py:transition_state():387] send defer: 8
+2022-02-03 18:07:40,174 DEBUG   HandlerThread:29156 [handler.py:handle_request():130] handle_request: defer
+2022-02-03 18:07:40,174 INFO    HandlerThread:29156 [handler.py:handle_request_defer():147] handle defer: 8
+2022-02-03 18:07:40,175 DEBUG   SenderThread:29156 [sender.py:send_request():248] send_request: defer
+2022-02-03 18:07:40,175 INFO    SenderThread:29156 [sender.py:send_request_defer():383] handle sender defer: 8
+2022-02-03 18:07:40,175 INFO    SenderThread:29156 [sender.py:transition_state():387] send defer: 9
+2022-02-03 18:07:40,176 DEBUG   SenderThread:29156 [sender.py:send():234] send: final
+2022-02-03 18:07:40,176 DEBUG   HandlerThread:29156 [handler.py:handle_request():130] handle_request: defer
+2022-02-03 18:07:40,176 INFO    HandlerThread:29156 [handler.py:handle_request_defer():147] handle defer: 9
+2022-02-03 18:07:40,176 DEBUG   SenderThread:29156 [sender.py:send_request():248] send_request: defer
+2022-02-03 18:07:40,177 INFO    SenderThread:29156 [sender.py:send_request_defer():383] handle sender defer: 9
+2022-02-03 18:07:40,271 DEBUG   HandlerThread:29156 [handler.py:handle_request():130] handle_request: poll_exit
+2022-02-03 18:07:40,272 DEBUG   SenderThread:29156 [sender.py:send_request():248] send_request: poll_exit
+2022-02-03 18:07:40,274 DEBUG   HandlerThread:29156 [handler.py:handle_request():130] handle_request: get_summary
+2022-02-03 18:07:40,380 DEBUG   HandlerThread:29156 [handler.py:handle_request():130] handle_request: sampled_history
+2022-02-03 18:07:40,383 DEBUG   HandlerThread:29156 [handler.py:handle_request():130] handle_request: shutdown
+2022-02-03 18:07:40,384 INFO    HandlerThread:29156 [handler.py:finish():731] shutting down handler
+2022-02-03 18:07:41,180 INFO    WriterThread:29156 [datastore.py:close():281] close: /workspace/wav2vec-cv7-1b-ir/wandb/offline-run-20220203_154548-23cvd7o7/run-23cvd7o7.wandb
+2022-02-03 18:07:41,273 INFO    SenderThread:29156 [sender.py:finish():1070] shutting down sender
+2022-02-03 18:07:41,281 INFO    MainThread:29156 [internal.py:handle_exit():77] Internal process exited

wandb/offline-run-20220203_154548-23cvd7o7/logs/debug.log

@@ -20,3 +20,15 @@ config: {}
 2022-02-03 15:45:50,377 INFO    MainThread:27059 [wandb_init.py:init():633] run started, returning control to user process
 2022-02-03 15:45:50,383 INFO    MainThread:27059 [wandb_run.py:_config_callback():956] config_cb None None {'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'float32', 'use_bfloat16': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'chunk_size_feed_forward': 0, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'architectures': ['Wav2Vec2ForPreTraining'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 1, 'pad_token_id': 31, 'eos_token_id': 2, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'facebook/wav2vec2-xls-r-1b', 'transformers_version': '4.16.0.dev0', 'feat_extract_dropout': 0.0, 'model_type': 'wav2vec2', 'num_feat_extract_layers': 7, 'hidden_size': 1280, 'feat_extract_norm': 'layer', 'feat_extract_activation': 'gelu', 'conv_dim': [512, 512, 512, 512, 512, 512, 512], 'conv_stride': [5, 2, 2, 2, 2, 2, 2], 'conv_kernel': [10, 3, 3, 3, 3, 2, 2], 'conv_bias': True, 'num_conv_pos_embeddings': 128, 'num_conv_pos_embedding_groups': 16, 'num_hidden_layers': 48, 'intermediate_size': 5120, 'hidden_act': 'gelu', 'num_attention_heads': 16, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'activation_dropout': 0.1, 'feat_proj_dropout': 0.0, 'final_dropout': 0.0, 'layerdrop': 0.0, 'layer_norm_eps': 1e-05, 'initializer_range': 0.02, 'vocab_size': 34, 'do_stable_layer_norm': True, 'use_weighted_layer_sum': False, 'apply_spec_augment': True, 'mask_time_prob': 0.75, 'mask_time_length': 10, 'mask_time_min_masks': 2, 'mask_feature_prob': 0.4, 'mask_feature_length': 64, 'mask_feature_min_masks': 0, 'num_codevectors_per_group': 320, 'num_codevector_groups': 2, 'contrastive_logits_temperature': 0.1, 'feat_quantizer_dropout': 0.0, 'num_negatives': 100, 'codevector_dim': 1024, 'proj_codevector_dim': 1024, 'diversity_loss_weight': 0.1, 'ctc_loss_reduction': 'mean', 'ctc_zero_infinity': False, 'add_adapter': False, 'adapter_kernel_size': 3, 'adapter_stride': 2, 'num_adapter_layers': 3, 'output_hidden_size': 1280, 'classifier_proj_size': 256, 'tdnn_dim': [512, 512, 512, 512, 1500], 'tdnn_kernel': [5, 3, 3, 1, 1], 'tdnn_dilation': [1, 2, 3, 1, 1], 'xvector_output_dim': 512, 'output_dir': './', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 16, 'per_device_eval_batch_size': 16, 'per_gpu_train_batch_size': 'None', 'per_gpu_eval_batch_size': 'None', 'gradient_accumulation_steps': 2, 'eval_accumulation_steps': 'None', 'learning_rate': 2e-05, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 100.0, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'warmup_ratio': 0.0, 'warmup_steps': 1000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Feb03_15-40-29_job-829e2c87-5501-41ef-ad65-a05e9b64bfd7', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 100, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 500, 'save_total_limit': 3, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'bf16': False, 'fp16': True, 'fp16_opt_level': 'O1', 'half_precision_backend': 'amp', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': 'None', 'local_rank': -1, 'xpu_backend': 'None', 'tpu_num_cores': 'None', 'tpu_metrics_debug': False, 'debug': '[]', 'dataloader_drop_last': False, 'eval_steps': 500, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': 'None', 'load_best_model_at_end': False, 'metric_for_best_model': 'None', 'greater_is_better': 'None', 'ignore_data_skip': False, 'sharded_ddp': '[]', 'deepspeed': 'None', 'label_smoothing_factor': 0.0, 'optim': 'adamw_hf', 'adafactor': False, 'group_by_length': True, 'length_column_name': 'input_length', 'report_to': "['tensorboard', 'wandb']", 'ddp_find_unused_parameters': 'None', 'ddp_bucket_cap_mb': 'None', 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': 'None', 'hub_model_id': 'None', 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'gradient_checkpointing': True, 'fp16_backend': 'auto', 'push_to_hub_model_id': 'None', 'push_to_hub_organization': 'None', 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', '_n_gpu': 1, 'mp_parameters': '', 'train_batch_size': 16, 'eval_batch_size': 16}
 2022-02-03 15:45:50,390 INFO    MainThread:27059 [wandb_watch.py:watch():43] Watching
+2022-02-03 18:07:37,273 INFO    MainThread:27059 [wandb_run.py:_atexit_cleanup():1780] got exitcode: 0
+2022-02-03 18:07:37,278 INFO    MainThread:27059 [wandb_run.py:_restore():1752] restore
+2022-02-03 18:07:40,034 INFO    MainThread:27059 [wandb_run.py:_wait_for_finish():1912] got exit ret: 
+2022-02-03 18:07:40,170 INFO    MainThread:27059 [wandb_run.py:_wait_for_finish():1912] got exit ret: 
+2022-02-03 18:07:40,273 INFO    MainThread:27059 [wandb_run.py:_wait_for_finish():1912] got exit ret: done: true
+exit_result {
+}
+local_info {
+}
+
+2022-02-03 18:07:41,456 INFO    MainThread:27059 [wandb_run.py:_append_history():2130] rendering history
+2022-02-03 18:07:41,457 INFO    MainThread:27059 [wandb_run.py:_append_summary():2085] rendering summary

wandb/offline-run-20220203_154548-23cvd7o7/run-23cvd7o7.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9bb35590405d0ad8e22d52e1228c23ec6aebc1d540d9016e15aca2d4e4843aef
-size 41575854
+oid sha256:5efdfd94d603c58a50259a019d4d2f809277b08b62ac0ad0f508b6fe193e2755
+size 42863793