Upload with huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,95 @@
+---
+tags:
+- text2text-generation
+- definition-modeling
+metrics:
+- rouge
+model-index:
+- name: mt0-definition-en-xl
+  results: []
+language:
+- en
+widget:
+- text: "He ate a sweet apple. What is the definition of apple?"
+  example_title: "Definition generation"
+- text: "The paper contains a number of original ideas about color perception. What is the definition of original?"
+  example_title: "Definition generation"
+license: cc-by-sa-4.0
+datasets:
+- marksverdhei/wordnet-definitions-en-2021
+---
+
+# mt0-definition-en-xl
+
+This model is a fine-tuned version of [mt0-xl/](https://huggingface.co/bigscience/mt0-xl) on English WordNet, CodWoE and Oxford.
+
+It achieves the following results on the evaluation set:
+- Loss: 1.7210
+- Rouge1: 41.5067
+- Rouge2: 23.7149
+- Rougel: 39.138
+- Rougelsum: 39.1647
+- Gen Len: 15.1578
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-05
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 128
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- num_epochs: 20.0
+
+### Training results
+
+| Training Loss | Epoch | Step  | Validation Loss | Rouge1  | Rouge2  | Rougel  | Rougelsum | Gen Len |
+|:-------------:|:-----:|:-----:|:---------------:|:-------:|:-------:|:-------:|:---------:|:-------:|
+| 2.1171        | 1.0   | 1370  | 1.8175          | 27.0261 | 8.6429  | 25.2826 | 25.2952   | 11.8798 |
+| 1.8186        | 2.0   | 2740  | 1.7112          | 29.1583 | 9.9747  | 27.3432 | 27.3647   | 11.7919 |
+| 1.643         | 3.0   | 4110  | 1.6442          | 30.9045 | 11.2256 | 28.7826 | 28.788    | 12.4125 |
+| 1.499         | 4.0   | 5480  | 1.5978          | 32.1126 | 12.6674 | 29.97   | 29.9843   | 12.3129 |
+| 1.3772        | 5.0   | 6850  | 1.5720          | 33.6113 | 13.8451 | 31.3468 | 31.3599   | 12.6887 |
+| 1.2742        | 6.0   | 8220  | 1.5564          | 34.4899 | 15.1005 | 32.3177 | 32.3291   | 12.2003 |
+| 1.1785        | 7.0   | 9590  | 1.5466          | 35.4729 | 16.2035 | 33.2166 | 33.2295   | 12.4487 |
+| 1.0941        | 8.0   | 10960 | 1.5571          | 36.4885 | 17.5396 | 34.2494 | 34.2759   | 12.7543 |
+| 1.0202        | 9.0   | 12330 | 1.5541          | 37.4019 | 18.5568 | 35.1341 | 35.1473   | 12.8603 |
+| 0.9552        | 10.0  | 13700 | 1.5642          | 38.127  | 19.4057 | 35.9008 | 35.9163   | 12.6987 |
+| 0.8963        | 11.0  | 15070 | 1.5772          | 38.5073 | 20.0584 | 36.3304 | 36.3399   | 12.7052 |
+| 0.8443        | 12.0  | 16440 | 1.5955          | 39.2323 | 20.9237 | 36.9863 | 37.0049   | 13.0395 |
+| 0.7982        | 13.0  | 17810 | 1.6089          | 39.7947 | 21.6422 | 37.5619 | 37.5815   | 13.1400 |
+| 0.7586        | 14.0  | 19180 | 1.6293          | 40.2922 | 22.2301 | 38.0755 | 38.0757   | 12.8589 |
+| 0.7234        | 15.0  | 20550 | 1.6493          | 40.6358 | 22.5355 | 38.3523 | 38.3659   | 13.1102 |
+| 0.6946        | 16.0  | 21920 | 1.6701          | 40.7708 | 22.906  | 38.5037 | 38.5174   | 13.1035 |
+| 0.6688        | 17.0  | 23290 | 1.6902          | 41.0847 | 23.1663 | 38.8126 | 38.8149   | 13.2951 |
+| 0.6484        | 18.0  | 24660 | 1.7005          | 41.2075 | 23.3967 | 38.9529 | 38.9545   | 13.2707 |
+| 0.6342        | 19.0  | 26030 | 1.7116          | 41.2454 | 23.5187 | 39.0203 | 39.0396   | 13.2173 |
+| 0.6234        | 20.0  | 27400 | 1.7210          | 41.3073 | 23.5691 | 39.0662 | 39.074    | 13.2558 |
+
+
+### Framework versions
+
+- Transformers 4.30.2
+- Pytorch 1.13.1+rocm5.2
+- Datasets 2.12.0
+- Tokenizers 0.12.1

all_results.json

@@ -0,0 +1,18 @@
+{
+    "epoch": 20.0,
+    "eval_gen_len": 15.15777428121871,
+    "eval_loss": 1.7210286855697632,
+    "eval_rouge1": 41.5067,
+    "eval_rouge2": 23.7149,
+    "eval_rougeL": 39.138,
+    "eval_rougeLsum": 39.1647,
+    "eval_runtime": 708.2379,
+    "eval_samples": 13982,
+    "eval_samples_per_second": 19.742,
+    "eval_steps_per_second": 0.617,
+    "train_loss": 1.0633717825116902,
+    "train_runtime": 55751.6068,
+    "train_samples": 175332,
+    "train_samples_per_second": 62.898,
+    "train_steps_per_second": 0.491
+}

config.json

@@ -0,0 +1,32 @@
+{
+  "_name_or_path": "/scratch/project_465000498/models/mt0-xl/",
+  "architectures": [
+    "MT5ForConditionalGeneration"
+  ],
+  "d_ff": 5120,
+  "d_kv": 64,
+  "d_model": 2048,
+  "decoder_start_token_id": 0,
+  "dense_act_fn": "gelu_new",
+  "dropout_rate": 0.1,
+  "eos_token_id": 1,
+  "feed_forward_proj": "gated-gelu",
+  "initializer_factor": 1.0,
+  "is_encoder_decoder": true,
+  "is_gated_act": true,
+  "layer_norm_epsilon": 1e-06,
+  "model_type": "mt5",
+  "num_decoder_layers": 24,
+  "num_heads": 32,
+  "num_layers": 24,
+  "output_past": true,
+  "pad_token_id": 0,
+  "relative_attention_max_distance": 128,
+  "relative_attention_num_buckets": 32,
+  "tie_word_embeddings": false,
+  "tokenizer_class": "T5Tokenizer",
+  "torch_dtype": "float32",
+  "transformers_version": "4.30.2",
+  "use_cache": true,
+  "vocab_size": 250112
+}

eval_results.json

@@ -0,0 +1,13 @@
+{
+    "epoch": 20.0,
+    "eval_gen_len": 15.15777428121871,
+    "eval_loss": 1.7210286855697632,
+    "eval_rouge1": 41.5067,
+    "eval_rouge2": 23.7149,
+    "eval_rougeL": 39.138,
+    "eval_rougeLsum": 39.1647,
+    "eval_runtime": 708.2379,
+    "eval_samples": 13982,
+    "eval_samples_per_second": 19.742,
+    "eval_steps_per_second": 0.617
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "decoder_start_token_id": 0,
+  "eos_token_id": 1,
+  "pad_token_id": 0,
+  "transformers_version": "4.30.2"
+}

pytorch_model-00001-of-00002.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1301013d25f963362f7e7d9b390be8914c2b8d54fdef7923879cf288faa44be5
+size 9977020596

pytorch_model-00002-of-00002.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7bd209e4a75c81b311ee9f923dab47d3ba2e4a927c748ce6ba55896dc7e39d0
+size 4993663292

pytorch_model.bin.index.json

@@ -0,0 +1,566 @@
+{
+  "metadata": {
+    "total_size": 14970478592
+  },
+  "weight_map": {
+    "decoder.block.0.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.0.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.0.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.0.layer.0.SelfAttention.relative_attention_bias.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.0.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.0.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.0.layer.1.EncDecAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.0.layer.1.EncDecAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.0.layer.1.EncDecAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.0.layer.1.EncDecAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.0.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.0.layer.2.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.0.layer.2.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.0.layer.2.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.0.layer.2.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.1.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.1.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.1.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.1.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.1.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.1.layer.1.EncDecAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.1.layer.1.EncDecAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.1.layer.1.EncDecAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.1.layer.1.EncDecAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.1.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.1.layer.2.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.1.layer.2.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.1.layer.2.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.1.layer.2.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.10.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.10.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.10.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.10.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.10.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.10.layer.1.EncDecAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.10.layer.1.EncDecAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.10.layer.1.EncDecAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.10.layer.1.EncDecAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.10.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.10.layer.2.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.10.layer.2.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.10.layer.2.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.10.layer.2.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.11.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.11.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.11.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.11.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.11.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.11.layer.1.EncDecAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.11.layer.1.EncDecAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.11.layer.1.EncDecAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.11.layer.1.EncDecAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.11.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.11.layer.2.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.11.layer.2.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.11.layer.2.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.11.layer.2.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.12.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.12.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.12.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.12.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.12.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.12.layer.1.EncDecAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.12.layer.1.EncDecAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.12.layer.1.EncDecAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.12.layer.1.EncDecAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.12.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.12.layer.2.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.12.layer.2.DenseReluDense.wi_1.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.12.layer.2.DenseReluDense.wo.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.12.layer.2.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.13.layer.0.SelfAttention.k.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.13.layer.0.SelfAttention.o.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.13.layer.0.SelfAttention.q.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.13.layer.0.SelfAttention.v.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.13.layer.0.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.13.layer.1.EncDecAttention.k.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.13.layer.1.EncDecAttention.o.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.13.layer.1.EncDecAttention.q.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.13.layer.1.EncDecAttention.v.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.13.layer.1.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.13.layer.2.DenseReluDense.wi_0.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.13.layer.2.DenseReluDense.wi_1.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.13.layer.2.DenseReluDense.wo.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.13.layer.2.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.14.layer.0.SelfAttention.k.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.14.layer.0.SelfAttention.o.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.14.layer.0.SelfAttention.q.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.14.layer.0.SelfAttention.v.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.14.layer.0.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.14.layer.1.EncDecAttention.k.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.14.layer.1.EncDecAttention.o.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.14.layer.1.EncDecAttention.q.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.14.layer.1.EncDecAttention.v.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.14.layer.1.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.14.layer.2.DenseReluDense.wi_0.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.14.layer.2.DenseReluDense.wi_1.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.14.layer.2.DenseReluDense.wo.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.14.layer.2.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.15.layer.0.SelfAttention.k.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.15.layer.0.SelfAttention.o.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.15.layer.0.SelfAttention.q.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.15.layer.0.SelfAttention.v.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.15.layer.0.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.15.layer.1.EncDecAttention.k.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.15.layer.1.EncDecAttention.o.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.15.layer.1.EncDecAttention.q.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.15.layer.1.EncDecAttention.v.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.15.layer.1.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.15.layer.2.DenseReluDense.wi_0.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.15.layer.2.DenseReluDense.wi_1.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.15.layer.2.DenseReluDense.wo.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.15.layer.2.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.16.layer.0.SelfAttention.k.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.16.layer.0.SelfAttention.o.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.16.layer.0.SelfAttention.q.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.16.layer.0.SelfAttention.v.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.16.layer.0.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.16.layer.1.EncDecAttention.k.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.16.layer.1.EncDecAttention.o.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.16.layer.1.EncDecAttention.q.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.16.layer.1.EncDecAttention.v.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.16.layer.1.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.16.layer.2.DenseReluDense.wi_0.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.16.layer.2.DenseReluDense.wi_1.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.16.layer.2.DenseReluDense.wo.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.16.layer.2.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.17.layer.0.SelfAttention.k.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.17.layer.0.SelfAttention.o.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.17.layer.0.SelfAttention.q.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.17.layer.0.SelfAttention.v.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.17.layer.0.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.17.layer.1.EncDecAttention.k.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.17.layer.1.EncDecAttention.o.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.17.layer.1.EncDecAttention.q.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.17.layer.1.EncDecAttention.v.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.17.layer.1.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.17.layer.2.DenseReluDense.wi_0.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.17.layer.2.DenseReluDense.wi_1.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.17.layer.2.DenseReluDense.wo.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.17.layer.2.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.18.layer.0.SelfAttention.k.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.18.layer.0.SelfAttention.o.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.18.layer.0.SelfAttention.q.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.18.layer.0.SelfAttention.v.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.18.layer.0.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.18.layer.1.EncDecAttention.k.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.18.layer.1.EncDecAttention.o.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.18.layer.1.EncDecAttention.q.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.18.layer.1.EncDecAttention.v.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.18.layer.1.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.18.layer.2.DenseReluDense.wi_0.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.18.layer.2.DenseReluDense.wi_1.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.18.layer.2.DenseReluDense.wo.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.18.layer.2.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.19.layer.0.SelfAttention.k.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.19.layer.0.SelfAttention.o.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.19.layer.0.SelfAttention.q.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.19.layer.0.SelfAttention.v.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.19.layer.0.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.19.layer.1.EncDecAttention.k.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.19.layer.1.EncDecAttention.o.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.19.layer.1.EncDecAttention.q.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.19.layer.1.EncDecAttention.v.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.19.layer.1.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.19.layer.2.DenseReluDense.wi_0.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.19.layer.2.DenseReluDense.wi_1.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.19.layer.2.DenseReluDense.wo.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.19.layer.2.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.2.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.2.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.2.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.2.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.2.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.2.layer.1.EncDecAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.2.layer.1.EncDecAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.2.layer.1.EncDecAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.2.layer.1.EncDecAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.2.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.2.layer.2.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.2.layer.2.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.2.layer.2.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.2.layer.2.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.20.layer.0.SelfAttention.k.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.20.layer.0.SelfAttention.o.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.20.layer.0.SelfAttention.q.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.20.layer.0.SelfAttention.v.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.20.layer.0.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.20.layer.1.EncDecAttention.k.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.20.layer.1.EncDecAttention.o.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.20.layer.1.EncDecAttention.q.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.20.layer.1.EncDecAttention.v.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.20.layer.1.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.20.layer.2.DenseReluDense.wi_0.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.20.layer.2.DenseReluDense.wi_1.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.20.layer.2.DenseReluDense.wo.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.20.layer.2.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.21.layer.0.SelfAttention.k.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.21.layer.0.SelfAttention.o.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.21.layer.0.SelfAttention.q.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.21.layer.0.SelfAttention.v.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.21.layer.0.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.21.layer.1.EncDecAttention.k.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.21.layer.1.EncDecAttention.o.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.21.layer.1.EncDecAttention.q.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.21.layer.1.EncDecAttention.v.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.21.layer.1.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.21.layer.2.DenseReluDense.wi_0.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.21.layer.2.DenseReluDense.wi_1.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.21.layer.2.DenseReluDense.wo.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.21.layer.2.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.22.layer.0.SelfAttention.k.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.22.layer.0.SelfAttention.o.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.22.layer.0.SelfAttention.q.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.22.layer.0.SelfAttention.v.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.22.layer.0.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.22.layer.1.EncDecAttention.k.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.22.layer.1.EncDecAttention.o.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.22.layer.1.EncDecAttention.q.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.22.layer.1.EncDecAttention.v.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.22.layer.1.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.22.layer.2.DenseReluDense.wi_0.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.22.layer.2.DenseReluDense.wi_1.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.22.layer.2.DenseReluDense.wo.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.22.layer.2.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.23.layer.0.SelfAttention.k.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.23.layer.0.SelfAttention.o.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.23.layer.0.SelfAttention.q.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.23.layer.0.SelfAttention.v.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.23.layer.0.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.23.layer.1.EncDecAttention.k.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.23.layer.1.EncDecAttention.o.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.23.layer.1.EncDecAttention.q.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.23.layer.1.EncDecAttention.v.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.23.layer.1.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.23.layer.2.DenseReluDense.wi_0.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.23.layer.2.DenseReluDense.wi_1.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.23.layer.2.DenseReluDense.wo.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.23.layer.2.layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "decoder.block.3.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.3.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.3.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.3.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.3.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.3.layer.1.EncDecAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.3.layer.1.EncDecAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.3.layer.1.EncDecAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.3.layer.1.EncDecAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.3.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.3.layer.2.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.3.layer.2.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.3.layer.2.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.3.layer.2.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.4.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.4.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.4.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.4.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.4.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.4.layer.1.EncDecAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.4.layer.1.EncDecAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.4.layer.1.EncDecAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.4.layer.1.EncDecAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.4.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.4.layer.2.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.4.layer.2.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.4.layer.2.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.4.layer.2.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.5.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.5.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.5.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.5.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.5.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.5.layer.1.EncDecAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.5.layer.1.EncDecAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.5.layer.1.EncDecAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.5.layer.1.EncDecAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.5.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.5.layer.2.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.5.layer.2.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.5.layer.2.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.5.layer.2.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.6.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.6.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.6.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.6.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.6.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.6.layer.1.EncDecAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.6.layer.1.EncDecAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.6.layer.1.EncDecAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.6.layer.1.EncDecAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.6.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.6.layer.2.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.6.layer.2.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.6.layer.2.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.6.layer.2.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.7.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.7.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.7.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.7.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.7.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.7.layer.1.EncDecAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.7.layer.1.EncDecAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.7.layer.1.EncDecAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.7.layer.1.EncDecAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.7.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.7.layer.2.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.7.layer.2.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.7.layer.2.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.7.layer.2.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.8.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.8.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.8.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.8.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.8.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.8.layer.1.EncDecAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.8.layer.1.EncDecAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.8.layer.1.EncDecAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.8.layer.1.EncDecAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.8.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.8.layer.2.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.8.layer.2.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.8.layer.2.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.8.layer.2.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.9.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.9.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.9.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.9.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.9.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.9.layer.1.EncDecAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.9.layer.1.EncDecAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.9.layer.1.EncDecAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.9.layer.1.EncDecAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.9.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.9.layer.2.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.9.layer.2.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.9.layer.2.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.block.9.layer.2.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.embed_tokens.weight": "pytorch_model-00001-of-00002.bin",
+    "decoder.final_layer_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "encoder.block.0.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.0.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.0.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.0.layer.0.SelfAttention.relative_attention_bias.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.0.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.0.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.0.layer.1.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.0.layer.1.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.0.layer.1.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.0.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.1.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.1.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.1.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.1.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.1.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.1.layer.1.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.1.layer.1.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.1.layer.1.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.1.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.10.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.10.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.10.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.10.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.10.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.10.layer.1.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.10.layer.1.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.10.layer.1.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.10.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.11.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.11.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.11.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.11.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.11.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.11.layer.1.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.11.layer.1.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.11.layer.1.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.11.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.12.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.12.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.12.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.12.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.12.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.12.layer.1.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.12.layer.1.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.12.layer.1.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.12.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.13.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.13.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.13.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.13.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.13.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.13.layer.1.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.13.layer.1.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.13.layer.1.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.13.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.14.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.14.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.14.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.14.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.14.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.14.layer.1.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.14.layer.1.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.14.layer.1.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.14.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.15.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.15.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.15.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.15.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.15.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.15.layer.1.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.15.layer.1.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.15.layer.1.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.15.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.16.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.16.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.16.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.16.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.16.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.16.layer.1.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.16.layer.1.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.16.layer.1.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.16.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.17.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.17.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.17.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.17.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.17.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.17.layer.1.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.17.layer.1.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.17.layer.1.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.17.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.18.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.18.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.18.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.18.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.18.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.18.layer.1.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.18.layer.1.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.18.layer.1.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.18.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.19.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.19.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.19.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.19.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.19.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.19.layer.1.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.19.layer.1.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.19.layer.1.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.19.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.2.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.2.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.2.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.2.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.2.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.2.layer.1.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.2.layer.1.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.2.layer.1.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.2.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.20.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.20.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.20.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.20.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.20.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.20.layer.1.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.20.layer.1.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.20.layer.1.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.20.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.21.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.21.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.21.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.21.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.21.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.21.layer.1.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.21.layer.1.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.21.layer.1.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.21.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.22.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.22.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.22.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.22.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.22.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.22.layer.1.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.22.layer.1.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.22.layer.1.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.22.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.23.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.23.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.23.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.23.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.23.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.23.layer.1.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.23.layer.1.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.23.layer.1.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.23.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.3.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.3.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.3.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.3.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.3.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.3.layer.1.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.3.layer.1.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.3.layer.1.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.3.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.4.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.4.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.4.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.4.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.4.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.4.layer.1.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.4.layer.1.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.4.layer.1.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.4.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.5.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.5.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.5.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.5.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.5.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.5.layer.1.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.5.layer.1.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.5.layer.1.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.5.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.6.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.6.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.6.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.6.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.6.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.6.layer.1.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.6.layer.1.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.6.layer.1.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.6.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.7.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.7.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.7.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.7.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.7.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.7.layer.1.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.7.layer.1.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.7.layer.1.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.7.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.8.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.8.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.8.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.8.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.8.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.8.layer.1.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.8.layer.1.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.8.layer.1.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.8.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.9.layer.0.SelfAttention.k.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.9.layer.0.SelfAttention.o.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.9.layer.0.SelfAttention.q.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.9.layer.0.SelfAttention.v.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.9.layer.0.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.9.layer.1.DenseReluDense.wi_0.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.9.layer.1.DenseReluDense.wi_1.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.9.layer.1.DenseReluDense.wo.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.block.9.layer.1.layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "encoder.final_layer_norm.weight": "pytorch_model-00001-of-00002.bin",
+    "lm_head.weight": "pytorch_model-00002-of-00002.bin",
+    "shared.weight": "pytorch_model-00001-of-00002.bin"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef78f86560d809067d12bac6c09f19a462cb3af3f54d2b8acbba26e1433125d6
+size 4309802

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6502d07619068a98aa2d3bb531332a694ffe108ca6c6fe62a467ccfe98d666b9
+size 16315219

tokenizer_config.json

@@ -0,0 +1,11 @@
+{
+  "additional_special_tokens": null,
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "</s>",
+  "extra_ids": 0,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "T5Tokenizer",
+  "unk_token": "<unk>"
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 20.0,
+    "train_loss": 1.0633717825116902,
+    "train_runtime": 55751.6068,
+    "train_samples": 175332,
+    "train_samples_per_second": 62.898,
+    "train_steps_per_second": 0.491
+}

trainer_state.json

@@ -0,0 +1,405 @@
+{
+  "best_metric": 41.3073,
+  "best_model_checkpoint": "models/mt0-xl_english_adafactor/checkpoint-27400",
+  "epoch": 20.0,
+  "global_step": 27400,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 1.0,
+      "learning_rate": 4.75e-05,
+      "loss": 2.1171,
+      "step": 1370
+    },
+    {
+      "epoch": 1.0,
+      "eval_gen_len": 11.879845515662995,
+      "eval_loss": 1.8174540996551514,
+      "eval_rouge1": 27.0261,
+      "eval_rouge2": 8.6429,
+      "eval_rougeL": 25.2826,
+      "eval_rougeLsum": 25.2952,
+      "eval_runtime": 356.3645,
+      "eval_samples_per_second": 39.235,
+      "eval_steps_per_second": 1.226,
+      "step": 1370
+    },
+    {
+      "epoch": 2.0,
+      "learning_rate": 4.5e-05,
+      "loss": 1.8186,
+      "step": 2740
+    },
+    {
+      "epoch": 2.0,
+      "eval_gen_len": 11.791946788728366,
+      "eval_loss": 1.7111847400665283,
+      "eval_rouge1": 29.1583,
+      "eval_rouge2": 9.9747,
+      "eval_rougeL": 27.3432,
+      "eval_rougeLsum": 27.3647,
+      "eval_runtime": 305.4337,
+      "eval_samples_per_second": 45.778,
+      "eval_steps_per_second": 1.431,
+      "step": 2740
+    },
+    {
+      "epoch": 3.0,
+      "learning_rate": 4.25e-05,
+      "loss": 1.643,
+      "step": 4110
+    },
+    {
+      "epoch": 3.0,
+      "eval_gen_len": 12.412530396223715,
+      "eval_loss": 1.6442108154296875,
+      "eval_rouge1": 30.9045,
+      "eval_rouge2": 11.2256,
+      "eval_rougeL": 28.7826,
+      "eval_rougeLsum": 28.788,
+      "eval_runtime": 308.7201,
+      "eval_samples_per_second": 45.29,
+      "eval_steps_per_second": 1.416,
+      "step": 4110
+    },
+    {
+      "epoch": 4.0,
+      "learning_rate": 4e-05,
+      "loss": 1.499,
+      "step": 5480
+    },
+    {
+      "epoch": 4.0,
+      "eval_gen_len": 12.31290230296095,
+      "eval_loss": 1.5977873802185059,
+      "eval_rouge1": 32.1126,
+      "eval_rouge2": 12.6674,
+      "eval_rougeL": 29.97,
+      "eval_rougeLsum": 29.9843,
+      "eval_runtime": 307.939,
+      "eval_samples_per_second": 45.405,
+      "eval_steps_per_second": 1.419,
+      "step": 5480
+    },
+    {
+      "epoch": 5.0,
+      "learning_rate": 3.7500000000000003e-05,
+      "loss": 1.3772,
+      "step": 6850
+    },
+    {
+      "epoch": 5.0,
+      "eval_gen_len": 12.688742669146045,
+      "eval_loss": 1.571955680847168,
+      "eval_rouge1": 33.6113,
+      "eval_rouge2": 13.8451,
+      "eval_rougeL": 31.3468,
+      "eval_rougeLsum": 31.3599,
+      "eval_runtime": 320.2331,
+      "eval_samples_per_second": 43.662,
+      "eval_steps_per_second": 1.365,
+      "step": 6850
+    },
+    {
+      "epoch": 6.0,
+      "learning_rate": 3.5e-05,
+      "loss": 1.2742,
+      "step": 8220
+    },
+    {
+      "epoch": 6.0,
+      "eval_gen_len": 12.200257473895007,
+      "eval_loss": 1.556434988975525,
+      "eval_rouge1": 34.4899,
+      "eval_rouge2": 15.1005,
+      "eval_rougeL": 32.3177,
+      "eval_rougeLsum": 32.3291,
+      "eval_runtime": 308.9469,
+      "eval_samples_per_second": 45.257,
+      "eval_steps_per_second": 1.414,
+      "step": 8220
+    },
+    {
+      "epoch": 7.0,
+      "learning_rate": 3.2500000000000004e-05,
+      "loss": 1.1785,
+      "step": 9590
+    },
+    {
+      "epoch": 7.0,
+      "eval_gen_len": 12.4487197825776,
+      "eval_loss": 1.5466481447219849,
+      "eval_rouge1": 35.4729,
+      "eval_rouge2": 16.2035,
+      "eval_rougeL": 33.2166,
+      "eval_rougeLsum": 33.2295,
+      "eval_runtime": 305.713,
+      "eval_samples_per_second": 45.736,
+      "eval_steps_per_second": 1.429,
+      "step": 9590
+    },
+    {
+      "epoch": 8.0,
+      "learning_rate": 3e-05,
+      "loss": 1.0941,
+      "step": 10960
+    },
+    {
+      "epoch": 8.0,
+      "eval_gen_len": 12.754255471320269,
+      "eval_loss": 1.5571105480194092,
+      "eval_rouge1": 36.4885,
+      "eval_rouge2": 17.5396,
+      "eval_rougeL": 34.2494,
+      "eval_rougeLsum": 34.2759,
+      "eval_runtime": 308.476,
+      "eval_samples_per_second": 45.326,
+      "eval_steps_per_second": 1.417,
+      "step": 10960
+    },
+    {
+      "epoch": 9.0,
+      "learning_rate": 2.7500000000000004e-05,
+      "loss": 1.0202,
+      "step": 12330
+    },
+    {
+      "epoch": 9.0,
+      "eval_gen_len": 12.860320411958233,
+      "eval_loss": 1.5540790557861328,
+      "eval_rouge1": 37.4019,
+      "eval_rouge2": 18.5568,
+      "eval_rougeL": 35.1341,
+      "eval_rougeLsum": 35.1473,
+      "eval_runtime": 308.6475,
+      "eval_samples_per_second": 45.301,
+      "eval_steps_per_second": 1.416,
+      "step": 12330
+    },
+    {
+      "epoch": 10.0,
+      "learning_rate": 2.5e-05,
+      "loss": 0.9552,
+      "step": 13700
+    },
+    {
+      "epoch": 10.0,
+      "eval_gen_len": 12.698684022314405,
+      "eval_loss": 1.5641505718231201,
+      "eval_rouge1": 38.127,
+      "eval_rouge2": 19.4057,
+      "eval_rougeL": 35.9008,
+      "eval_rougeLsum": 35.9163,
+      "eval_runtime": 307.6813,
+      "eval_samples_per_second": 45.443,
+      "eval_steps_per_second": 1.42,
+      "step": 13700
+    },
+    {
+      "epoch": 11.0,
+      "learning_rate": 2.25e-05,
+      "loss": 0.8963,
+      "step": 15070
+    },
+    {
+      "epoch": 11.0,
+      "eval_gen_len": 12.705192390215991,
+      "eval_loss": 1.5771721601486206,
+      "eval_rouge1": 38.5073,
+      "eval_rouge2": 20.0584,
+      "eval_rougeL": 36.3304,
+      "eval_rougeLsum": 36.3399,
+      "eval_runtime": 480.883,
+      "eval_samples_per_second": 29.076,
+      "eval_steps_per_second": 0.909,
+      "step": 15070
+    },
+    {
+      "epoch": 12.0,
+      "learning_rate": 2e-05,
+      "loss": 0.8443,
+      "step": 16440
+    },
+    {
+      "epoch": 12.0,
+      "eval_gen_len": 13.039479330567874,
+      "eval_loss": 1.595460057258606,
+      "eval_rouge1": 39.2323,
+      "eval_rouge2": 20.9237,
+      "eval_rougeL": 36.9863,
+      "eval_rougeLsum": 37.0049,
+      "eval_runtime": 314.6611,
+      "eval_samples_per_second": 44.435,
+      "eval_steps_per_second": 1.389,
+      "step": 16440
+    },
+    {
+      "epoch": 13.0,
+      "learning_rate": 1.75e-05,
+      "loss": 0.7982,
+      "step": 17810
+    },
+    {
+      "epoch": 13.0,
+      "eval_gen_len": 13.140037190673723,
+      "eval_loss": 1.608866810798645,
+      "eval_rouge1": 39.7947,
+      "eval_rouge2": 21.6422,
+      "eval_rougeL": 37.5619,
+      "eval_rougeLsum": 37.5815,
+      "eval_runtime": 465.3407,
+      "eval_samples_per_second": 30.047,
+      "eval_steps_per_second": 0.939,
+      "step": 17810
+    },
+    {
+      "epoch": 14.0,
+      "learning_rate": 1.5e-05,
+      "loss": 0.7586,
+      "step": 19180
+    },
+    {
+      "epoch": 14.0,
+      "eval_gen_len": 12.85889000143041,
+      "eval_loss": 1.6293412446975708,
+      "eval_rouge1": 40.2922,
+      "eval_rouge2": 22.2301,
+      "eval_rougeL": 38.0755,
+      "eval_rougeLsum": 38.0757,
+      "eval_runtime": 312.4581,
+      "eval_samples_per_second": 44.748,
+      "eval_steps_per_second": 1.399,
+      "step": 19180
+    },
+    {
+      "epoch": 15.0,
+      "learning_rate": 1.25e-05,
+      "loss": 0.7234,
+      "step": 20550
+    },
+    {
+      "epoch": 15.0,
+      "eval_gen_len": 13.110213131168646,
+      "eval_loss": 1.6492763757705688,
+      "eval_rouge1": 40.6358,
+      "eval_rouge2": 22.5355,
+      "eval_rougeL": 38.3523,
+      "eval_rougeLsum": 38.3659,
+      "eval_runtime": 307.5457,
+      "eval_samples_per_second": 45.463,
+      "eval_steps_per_second": 1.421,
+      "step": 20550
+    },
+    {
+      "epoch": 16.0,
+      "learning_rate": 1e-05,
+      "loss": 0.6946,
+      "step": 21920
+    },
+    {
+      "epoch": 16.0,
+      "eval_gen_len": 13.103490201687885,
+      "eval_loss": 1.6700669527053833,
+      "eval_rouge1": 40.7708,
+      "eval_rouge2": 22.906,
+      "eval_rougeL": 38.5037,
+      "eval_rougeLsum": 38.5174,
+      "eval_runtime": 306.4464,
+      "eval_samples_per_second": 45.626,
+      "eval_steps_per_second": 1.426,
+      "step": 21920
+    },
+    {
+      "epoch": 17.0,
+      "learning_rate": 7.5e-06,
+      "loss": 0.6688,
+      "step": 23290
+    },
+    {
+      "epoch": 17.0,
+      "eval_gen_len": 13.295093691889573,
+      "eval_loss": 1.690183162689209,
+      "eval_rouge1": 41.0847,
+      "eval_rouge2": 23.1663,
+      "eval_rougeL": 38.8126,
+      "eval_rougeLsum": 38.8149,
+      "eval_runtime": 309.6148,
+      "eval_samples_per_second": 45.159,
+      "eval_steps_per_second": 1.411,
+      "step": 23290
+    },
+    {
+      "epoch": 18.0,
+      "learning_rate": 5e-06,
+      "loss": 0.6484,
+      "step": 24660
+    },
+    {
+      "epoch": 18.0,
+      "eval_gen_len": 13.270705192390215,
+      "eval_loss": 1.7005170583724976,
+      "eval_rouge1": 41.2075,
+      "eval_rouge2": 23.3967,
+      "eval_rougeL": 38.9529,
+      "eval_rougeLsum": 38.9545,
+      "eval_runtime": 310.6615,
+      "eval_samples_per_second": 45.007,
+      "eval_steps_per_second": 1.407,
+      "step": 24660
+    },
+    {
+      "epoch": 19.0,
+      "learning_rate": 2.5e-06,
+      "loss": 0.6342,
+      "step": 26030
+    },
+    {
+      "epoch": 19.0,
+      "eval_gen_len": 13.217279359176084,
+      "eval_loss": 1.7115505933761597,
+      "eval_rouge1": 41.2454,
+      "eval_rouge2": 23.5187,
+      "eval_rougeL": 39.0203,
+      "eval_rougeLsum": 39.0396,
+      "eval_runtime": 310.561,
+      "eval_samples_per_second": 45.022,
+      "eval_steps_per_second": 1.407,
+      "step": 26030
+    },
+    {
+      "epoch": 20.0,
+      "learning_rate": 0.0,
+      "loss": 0.6234,
+      "step": 27400
+    },
+    {
+      "epoch": 20.0,
+      "eval_gen_len": 13.255757402374481,
+      "eval_loss": 1.7210286855697632,
+      "eval_rouge1": 41.3073,
+      "eval_rouge2": 23.5691,
+      "eval_rougeL": 39.0662,
+      "eval_rougeLsum": 39.074,
+      "eval_runtime": 482.7425,
+      "eval_samples_per_second": 28.964,
+      "eval_steps_per_second": 0.905,
+      "step": 27400
+    },
+    {
+      "epoch": 20.0,
+      "step": 27400,
+      "total_flos": 4.389256654374306e+18,
+      "train_loss": 1.0633717825116902,
+      "train_runtime": 55751.6068,
+      "train_samples_per_second": 62.898,
+      "train_steps_per_second": 0.491
+    }
+  ],
+  "max_steps": 27400,
+  "num_train_epochs": 20,
+  "total_flos": 4.389256654374306e+18,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2de4920012d9a05879bae5eda49277823930d531dd6e4aa4ec826cf427402a70
+size 4091