Upload 8 files

config.json

@@ -0,0 +1,25 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 128,
+  "max_position_embeddings": 128,
+  "model_type": "llama",
+  "num_attention_heads": 8,
+  "num_hidden_layers": 8,
+  "num_key_value_heads": 8,
+  "pad_token_id": 1,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.32.1",
+  "use_cache": true,
+  "vocab_size": 16000
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "pad_token_id": 1,
+  "transformers_version": "4.32.1"
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:600ec9802331fc7a9e68b938467c7ac4a02c57eebe4f15d2664ed7f9d3088708
+size 20087695

results.json

@@ -0,0 +1,2247 @@
+{
+  "results": {
+    "blimp": {
+      "acc,none": 0.6665074626865671,
+      "acc_stderr,none": 0.2001790545943022,
+      "alias": "blimp"
+    },
+    "blimp_adjunct_island": {
+      "acc,none": 0.668,
+      "acc_stderr,none": 0.014899597242811476,
+      "alias": " - blimp_adjunct_island"
+    },
+    "blimp_anaphor_gender_agreement": {
+      "acc,none": 0.67,
+      "acc_stderr,none": 0.014876872027456741,
+      "alias": " - blimp_anaphor_gender_agreement"
+    },
+    "blimp_anaphor_number_agreement": {
+      "acc,none": 0.863,
+      "acc_stderr,none": 0.01087884871433333,
+      "alias": " - blimp_anaphor_number_agreement"
+    },
+    "blimp_animate_subject_passive": {
+      "acc,none": 0.74,
+      "acc_stderr,none": 0.013877773329774166,
+      "alias": " - blimp_animate_subject_passive"
+    },
+    "blimp_animate_subject_trans": {
+      "acc,none": 0.692,
+      "acc_stderr,none": 0.01460648312734276,
+      "alias": " - blimp_animate_subject_trans"
+    },
+    "blimp_causative": {
+      "acc,none": 0.613,
+      "acc_stderr,none": 0.015410011955493935,
+      "alias": " - blimp_causative"
+    },
+    "blimp_complex_NP_island": {
+      "acc,none": 0.539,
+      "acc_stderr,none": 0.01577110420128319,
+      "alias": " - blimp_complex_NP_island"
+    },
+    "blimp_coordinate_structure_constraint_complex_left_branch": {
+      "acc,none": 0.378,
+      "acc_stderr,none": 0.015341165254026649,
+      "alias": " - blimp_coordinate_structure_constraint_complex_left_branch"
+    },
+    "blimp_coordinate_structure_constraint_object_extraction": {
+      "acc,none": 0.559,
+      "acc_stderr,none": 0.015708779894242676,
+      "alias": " - blimp_coordinate_structure_constraint_object_extraction"
+    },
+    "blimp_determiner_noun_agreement_1": {
+      "acc,none": 0.912,
+      "acc_stderr,none": 0.008963053962592086,
+      "alias": " - blimp_determiner_noun_agreement_1"
+    },
+    "blimp_determiner_noun_agreement_2": {
+      "acc,none": 0.932,
+      "acc_stderr,none": 0.007964887911291605,
+      "alias": " - blimp_determiner_noun_agreement_2"
+    },
+    "blimp_determiner_noun_agreement_irregular_1": {
+      "acc,none": 0.738,
+      "acc_stderr,none": 0.013912208651021345,
+      "alias": " - blimp_determiner_noun_agreement_irregular_1"
+    },
+    "blimp_determiner_noun_agreement_irregular_2": {
+      "acc,none": 0.849,
+      "acc_stderr,none": 0.011328165223341681,
+      "alias": " - blimp_determiner_noun_agreement_irregular_2"
+    },
+    "blimp_determiner_noun_agreement_with_adj_2": {
+      "acc,none": 0.885,
+      "acc_stderr,none": 0.010093407594904624,
+      "alias": " - blimp_determiner_noun_agreement_with_adj_2"
+    },
+    "blimp_determiner_noun_agreement_with_adj_irregular_1": {
+      "acc,none": 0.787,
+      "acc_stderr,none": 0.01295371756673723,
+      "alias": " - blimp_determiner_noun_agreement_with_adj_irregular_1"
+    },
+    "blimp_determiner_noun_agreement_with_adj_irregular_2": {
+      "acc,none": 0.826,
+      "acc_stderr,none": 0.011994493230973438,
+      "alias": " - blimp_determiner_noun_agreement_with_adj_irregular_2"
+    },
+    "blimp_determiner_noun_agreement_with_adjective_1": {
+      "acc,none": 0.888,
+      "acc_stderr,none": 0.00997775303139724,
+      "alias": " - blimp_determiner_noun_agreement_with_adjective_1"
+    },
+    "blimp_distractor_agreement_relational_noun": {
+      "acc,none": 0.446,
+      "acc_stderr,none": 0.015726771166750357,
+      "alias": " - blimp_distractor_agreement_relational_noun"
+    },
+    "blimp_distractor_agreement_relative_clause": {
+      "acc,none": 0.412,
+      "acc_stderr,none": 0.01557236329201509,
+      "alias": " - blimp_distractor_agreement_relative_clause"
+    },
+    "blimp_drop_argument": {
+      "acc,none": 0.69,
+      "acc_stderr,none": 0.014632638658632902,
+      "alias": " - blimp_drop_argument"
+    },
+    "blimp_ellipsis_n_bar_1": {
+      "acc,none": 0.599,
+      "acc_stderr,none": 0.015506109745498316,
+      "alias": " - blimp_ellipsis_n_bar_1"
+    },
+    "blimp_ellipsis_n_bar_2": {
+      "acc,none": 0.483,
+      "acc_stderr,none": 0.015810153729833434,
+      "alias": " - blimp_ellipsis_n_bar_2"
+    },
+    "blimp_existential_there_object_raising": {
+      "acc,none": 0.696,
+      "acc_stderr,none": 0.014553205687950427,
+      "alias": " - blimp_existential_there_object_raising"
+    },
+    "blimp_existential_there_quantifiers_1": {
+      "acc,none": 0.936,
+      "acc_stderr,none": 0.007743640226919291,
+      "alias": " - blimp_existential_there_quantifiers_1"
+    },
+    "blimp_existential_there_quantifiers_2": {
+      "acc,none": 0.488,
+      "acc_stderr,none": 0.015814743314581818,
+      "alias": " - blimp_existential_there_quantifiers_2"
+    },
+    "blimp_existential_there_subject_raising": {
+      "acc,none": 0.766,
+      "acc_stderr,none": 0.013394902889660006,
+      "alias": " - blimp_existential_there_subject_raising"
+    },
+    "blimp_expletive_it_object_raising": {
+      "acc,none": 0.661,
+      "acc_stderr,none": 0.014976758771620344,
+      "alias": " - blimp_expletive_it_object_raising"
+    },
+    "blimp_inchoative": {
+      "acc,none": 0.51,
+      "acc_stderr,none": 0.015816135752773203,
+      "alias": " - blimp_inchoative"
+    },
+    "blimp_intransitive": {
+      "acc,none": 0.648,
+      "acc_stderr,none": 0.015110404505648675,
+      "alias": " - blimp_intransitive"
+    },
+    "blimp_irregular_past_participle_adjectives": {
+      "acc,none": 0.811,
+      "acc_stderr,none": 0.012386784588117709,
+      "alias": " - blimp_irregular_past_participle_adjectives"
+    },
+    "blimp_irregular_past_participle_verbs": {
+      "acc,none": 0.819,
+      "acc_stderr,none": 0.01218143617917792,
+      "alias": " - blimp_irregular_past_participle_verbs"
+    },
+    "blimp_irregular_plural_subject_verb_agreement_1": {
+      "acc,none": 0.698,
+      "acc_stderr,none": 0.014526080235459543,
+      "alias": " - blimp_irregular_plural_subject_verb_agreement_1"
+    },
+    "blimp_irregular_plural_subject_verb_agreement_2": {
+      "acc,none": 0.779,
+      "acc_stderr,none": 0.01312750285969623,
+      "alias": " - blimp_irregular_plural_subject_verb_agreement_2"
+    },
+    "blimp_left_branch_island_echo_question": {
+      "acc,none": 0.542,
+      "acc_stderr,none": 0.015763390640483703,
+      "alias": " - blimp_left_branch_island_echo_question"
+    },
+    "blimp_left_branch_island_simple_question": {
+      "acc,none": 0.496,
+      "acc_stderr,none": 0.01581879370351089,
+      "alias": " - blimp_left_branch_island_simple_question"
+    },
+    "blimp_matrix_question_npi_licensor_present": {
+      "acc,none": 0.053,
+      "acc_stderr,none": 0.007088105617246444,
+      "alias": " - blimp_matrix_question_npi_licensor_present"
+    },
+    "blimp_npi_present_1": {
+      "acc,none": 0.327,
+      "acc_stderr,none": 0.014842213153411247,
+      "alias": " - blimp_npi_present_1"
+    },
+    "blimp_npi_present_2": {
+      "acc,none": 0.477,
+      "acc_stderr,none": 0.015802554246726098,
+      "alias": " - blimp_npi_present_2"
+    },
+    "blimp_only_npi_licensor_present": {
+      "acc,none": 0.805,
+      "acc_stderr,none": 0.012535235623319332,
+      "alias": " - blimp_only_npi_licensor_present"
+    },
+    "blimp_only_npi_scope": {
+      "acc,none": 0.569,
+      "acc_stderr,none": 0.015667944488173505,
+      "alias": " - blimp_only_npi_scope"
+    },
+    "blimp_passive_1": {
+      "acc,none": 0.832,
+      "acc_stderr,none": 0.011828605831454259,
+      "alias": " - blimp_passive_1"
+    },
+    "blimp_passive_2": {
+      "acc,none": 0.799,
+      "acc_stderr,none": 0.012679107214617331,
+      "alias": " - blimp_passive_2"
+    },
+    "blimp_principle_A_c_command": {
+      "acc,none": 0.573,
+      "acc_stderr,none": 0.015649789644462214,
+      "alias": " - blimp_principle_A_c_command"
+    },
+    "blimp_principle_A_case_1": {
+      "acc,none": 1.0,
+      "acc_stderr,none": 0.0,
+      "alias": " - blimp_principle_A_case_1"
+    },
+    "blimp_principle_A_case_2": {
+      "acc,none": 0.819,
+      "acc_stderr,none": 0.012181436179177912,
+      "alias": " - blimp_principle_A_case_2"
+    },
+    "blimp_principle_A_domain_1": {
+      "acc,none": 0.967,
+      "acc_stderr,none": 0.005651808820452374,
+      "alias": " - blimp_principle_A_domain_1"
+    },
+    "blimp_principle_A_domain_2": {
+      "acc,none": 0.66,
+      "acc_stderr,none": 0.014987482264363937,
+      "alias": " - blimp_principle_A_domain_2"
+    },
+    "blimp_principle_A_domain_3": {
+      "acc,none": 0.513,
+      "acc_stderr,none": 0.015813952101896626,
+      "alias": " - blimp_principle_A_domain_3"
+    },
+    "blimp_principle_A_reconstruction": {
+      "acc,none": 0.365,
+      "acc_stderr,none": 0.015231776226264891,
+      "alias": " - blimp_principle_A_reconstruction"
+    },
+    "blimp_regular_plural_subject_verb_agreement_1": {
+      "acc,none": 0.818,
+      "acc_stderr,none": 0.012207580637662164,
+      "alias": " - blimp_regular_plural_subject_verb_agreement_1"
+    },
+    "blimp_regular_plural_subject_verb_agreement_2": {
+      "acc,none": 0.693,
+      "acc_stderr,none": 0.01459328489285263,
+      "alias": " - blimp_regular_plural_subject_verb_agreement_2"
+    },
+    "blimp_sentential_negation_npi_licensor_present": {
+      "acc,none": 0.991,
+      "acc_stderr,none": 0.002987963843142665,
+      "alias": " - blimp_sentential_negation_npi_licensor_present"
+    },
+    "blimp_sentential_negation_npi_scope": {
+      "acc,none": 0.426,
+      "acc_stderr,none": 0.015645087688113814,
+      "alias": " - blimp_sentential_negation_npi_scope"
+    },
+    "blimp_sentential_subject_island": {
+      "acc,none": 0.451,
+      "acc_stderr,none": 0.015743152379585533,
+      "alias": " - blimp_sentential_subject_island"
+    },
+    "blimp_superlative_quantifiers_1": {
+      "acc,none": 0.834,
+      "acc_stderr,none": 0.011772110370812194,
+      "alias": " - blimp_superlative_quantifiers_1"
+    },
+    "blimp_superlative_quantifiers_2": {
+      "acc,none": 0.682,
+      "acc_stderr,none": 0.014734079309311901,
+      "alias": " - blimp_superlative_quantifiers_2"
+    },
+    "blimp_tough_vs_raising_1": {
+      "acc,none": 0.348,
+      "acc_stderr,none": 0.01507060460376841,
+      "alias": " - blimp_tough_vs_raising_1"
+    },
+    "blimp_tough_vs_raising_2": {
+      "acc,none": 0.806,
+      "acc_stderr,none": 0.012510816141264359,
+      "alias": " - blimp_tough_vs_raising_2"
+    },
+    "blimp_transitive": {
+      "acc,none": 0.721,
+      "acc_stderr,none": 0.014190150117612045,
+      "alias": " - blimp_transitive"
+    },
+    "blimp_wh_island": {
+      "acc,none": 0.498,
+      "acc_stderr,none": 0.015819173374302706,
+      "alias": " - blimp_wh_island"
+    },
+    "blimp_wh_questions_object_gap": {
+      "acc,none": 0.588,
+      "acc_stderr,none": 0.015572363292015095,
+      "alias": " - blimp_wh_questions_object_gap"
+    },
+    "blimp_wh_questions_subject_gap": {
+      "acc,none": 0.872,
+      "acc_stderr,none": 0.010570133761108663,
+      "alias": " - blimp_wh_questions_subject_gap"
+    },
+    "blimp_wh_questions_subject_gap_long_distance": {
+      "acc,none": 0.923,
+      "acc_stderr,none": 0.008434580140240648,
+      "alias": " - blimp_wh_questions_subject_gap_long_distance"
+    },
+    "blimp_wh_vs_that_no_gap": {
+      "acc,none": 0.92,
+      "acc_stderr,none": 0.00858333697775365,
+      "alias": " - blimp_wh_vs_that_no_gap"
+    },
+    "blimp_wh_vs_that_no_gap_long_distance": {
+      "acc,none": 0.962,
+      "acc_stderr,none": 0.0060491811505849384,
+      "alias": " - blimp_wh_vs_that_no_gap_long_distance"
+    },
+    "blimp_wh_vs_that_with_gap": {
+      "acc,none": 0.26,
+      "acc_stderr,none": 0.013877773329774164,
+      "alias": " - blimp_wh_vs_that_with_gap"
+    },
+    "blimp_wh_vs_that_with_gap_long_distance": {
+      "acc,none": 0.085,
+      "acc_stderr,none": 0.008823426366942288,
+      "alias": " - blimp_wh_vs_that_with_gap_long_distance"
+    }
+  },
+  "groups": {
+    "blimp": {
+      "acc,none": 0.6665074626865671,
+      "acc_stderr,none": 0.2001790545943022,
+      "alias": "blimp"
+    }
+  },
+  "configs": {
+    "blimp_adjunct_island": {
+      "task": "blimp_adjunct_island",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "adjunct_island",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_anaphor_gender_agreement": {
+      "task": "blimp_anaphor_gender_agreement",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "anaphor_gender_agreement",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_anaphor_number_agreement": {
+      "task": "blimp_anaphor_number_agreement",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "anaphor_number_agreement",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_animate_subject_passive": {
+      "task": "blimp_animate_subject_passive",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "animate_subject_passive",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_animate_subject_trans": {
+      "task": "blimp_animate_subject_trans",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "animate_subject_trans",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_causative": {
+      "task": "blimp_causative",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "causative",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_complex_NP_island": {
+      "task": "blimp_complex_NP_island",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "complex_NP_island",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_coordinate_structure_constraint_complex_left_branch": {
+      "task": "blimp_coordinate_structure_constraint_complex_left_branch",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "coordinate_structure_constraint_complex_left_branch",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_coordinate_structure_constraint_object_extraction": {
+      "task": "blimp_coordinate_structure_constraint_object_extraction",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "coordinate_structure_constraint_object_extraction",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_determiner_noun_agreement_1": {
+      "task": "blimp_determiner_noun_agreement_1",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "determiner_noun_agreement_1",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_determiner_noun_agreement_2": {
+      "task": "blimp_determiner_noun_agreement_2",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "determiner_noun_agreement_2",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_determiner_noun_agreement_irregular_1": {
+      "task": "blimp_determiner_noun_agreement_irregular_1",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "determiner_noun_agreement_irregular_1",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_determiner_noun_agreement_irregular_2": {
+      "task": "blimp_determiner_noun_agreement_irregular_2",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "determiner_noun_agreement_irregular_2",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_determiner_noun_agreement_with_adj_2": {
+      "task": "blimp_determiner_noun_agreement_with_adj_2",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "determiner_noun_agreement_with_adj_2",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_determiner_noun_agreement_with_adj_irregular_1": {
+      "task": "blimp_determiner_noun_agreement_with_adj_irregular_1",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "determiner_noun_agreement_with_adj_irregular_1",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_determiner_noun_agreement_with_adj_irregular_2": {
+      "task": "blimp_determiner_noun_agreement_with_adj_irregular_2",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "determiner_noun_agreement_with_adj_irregular_2",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_determiner_noun_agreement_with_adjective_1": {
+      "task": "blimp_determiner_noun_agreement_with_adjective_1",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "determiner_noun_agreement_with_adjective_1",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_distractor_agreement_relational_noun": {
+      "task": "blimp_distractor_agreement_relational_noun",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "distractor_agreement_relational_noun",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_distractor_agreement_relative_clause": {
+      "task": "blimp_distractor_agreement_relative_clause",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "distractor_agreement_relative_clause",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_drop_argument": {
+      "task": "blimp_drop_argument",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "drop_argument",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_ellipsis_n_bar_1": {
+      "task": "blimp_ellipsis_n_bar_1",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "ellipsis_n_bar_1",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_ellipsis_n_bar_2": {
+      "task": "blimp_ellipsis_n_bar_2",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "ellipsis_n_bar_2",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_existential_there_object_raising": {
+      "task": "blimp_existential_there_object_raising",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "existential_there_object_raising",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_existential_there_quantifiers_1": {
+      "task": "blimp_existential_there_quantifiers_1",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "existential_there_quantifiers_1",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_existential_there_quantifiers_2": {
+      "task": "blimp_existential_there_quantifiers_2",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "existential_there_quantifiers_2",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_existential_there_subject_raising": {
+      "task": "blimp_existential_there_subject_raising",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "existential_there_subject_raising",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_expletive_it_object_raising": {
+      "task": "blimp_expletive_it_object_raising",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "expletive_it_object_raising",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_inchoative": {
+      "task": "blimp_inchoative",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "inchoative",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_intransitive": {
+      "task": "blimp_intransitive",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "intransitive",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_irregular_past_participle_adjectives": {
+      "task": "blimp_irregular_past_participle_adjectives",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "irregular_past_participle_adjectives",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_irregular_past_participle_verbs": {
+      "task": "blimp_irregular_past_participle_verbs",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "irregular_past_participle_verbs",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_irregular_plural_subject_verb_agreement_1": {
+      "task": "blimp_irregular_plural_subject_verb_agreement_1",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "irregular_plural_subject_verb_agreement_1",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_irregular_plural_subject_verb_agreement_2": {
+      "task": "blimp_irregular_plural_subject_verb_agreement_2",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "irregular_plural_subject_verb_agreement_2",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_left_branch_island_echo_question": {
+      "task": "blimp_left_branch_island_echo_question",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "left_branch_island_echo_question",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_left_branch_island_simple_question": {
+      "task": "blimp_left_branch_island_simple_question",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "left_branch_island_simple_question",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_matrix_question_npi_licensor_present": {
+      "task": "blimp_matrix_question_npi_licensor_present",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "matrix_question_npi_licensor_present",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_npi_present_1": {
+      "task": "blimp_npi_present_1",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "npi_present_1",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_npi_present_2": {
+      "task": "blimp_npi_present_2",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "npi_present_2",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_only_npi_licensor_present": {
+      "task": "blimp_only_npi_licensor_present",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "only_npi_licensor_present",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_only_npi_scope": {
+      "task": "blimp_only_npi_scope",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "only_npi_scope",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_passive_1": {
+      "task": "blimp_passive_1",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "passive_1",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_passive_2": {
+      "task": "blimp_passive_2",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "passive_2",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_principle_A_c_command": {
+      "task": "blimp_principle_A_c_command",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "principle_A_c_command",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_principle_A_case_1": {
+      "task": "blimp_principle_A_case_1",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "principle_A_case_1",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_principle_A_case_2": {
+      "task": "blimp_principle_A_case_2",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "principle_A_case_2",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_principle_A_domain_1": {
+      "task": "blimp_principle_A_domain_1",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "principle_A_domain_1",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_principle_A_domain_2": {
+      "task": "blimp_principle_A_domain_2",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "principle_A_domain_2",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_principle_A_domain_3": {
+      "task": "blimp_principle_A_domain_3",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "principle_A_domain_3",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_principle_A_reconstruction": {
+      "task": "blimp_principle_A_reconstruction",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "principle_A_reconstruction",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_regular_plural_subject_verb_agreement_1": {
+      "task": "blimp_regular_plural_subject_verb_agreement_1",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "regular_plural_subject_verb_agreement_1",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_regular_plural_subject_verb_agreement_2": {
+      "task": "blimp_regular_plural_subject_verb_agreement_2",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "regular_plural_subject_verb_agreement_2",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_sentential_negation_npi_licensor_present": {
+      "task": "blimp_sentential_negation_npi_licensor_present",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "sentential_negation_npi_licensor_present",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_sentential_negation_npi_scope": {
+      "task": "blimp_sentential_negation_npi_scope",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "sentential_negation_npi_scope",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_sentential_subject_island": {
+      "task": "blimp_sentential_subject_island",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "sentential_subject_island",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_superlative_quantifiers_1": {
+      "task": "blimp_superlative_quantifiers_1",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "superlative_quantifiers_1",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_superlative_quantifiers_2": {
+      "task": "blimp_superlative_quantifiers_2",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "superlative_quantifiers_2",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_tough_vs_raising_1": {
+      "task": "blimp_tough_vs_raising_1",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "tough_vs_raising_1",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_tough_vs_raising_2": {
+      "task": "blimp_tough_vs_raising_2",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "tough_vs_raising_2",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_transitive": {
+      "task": "blimp_transitive",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "transitive",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_wh_island": {
+      "task": "blimp_wh_island",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "wh_island",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_wh_questions_object_gap": {
+      "task": "blimp_wh_questions_object_gap",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "wh_questions_object_gap",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_wh_questions_subject_gap": {
+      "task": "blimp_wh_questions_subject_gap",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "wh_questions_subject_gap",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_wh_questions_subject_gap_long_distance": {
+      "task": "blimp_wh_questions_subject_gap_long_distance",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "wh_questions_subject_gap_long_distance",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_wh_vs_that_no_gap": {
+      "task": "blimp_wh_vs_that_no_gap",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "wh_vs_that_no_gap",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_wh_vs_that_no_gap_long_distance": {
+      "task": "blimp_wh_vs_that_no_gap_long_distance",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "wh_vs_that_no_gap_long_distance",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_wh_vs_that_with_gap": {
+      "task": "blimp_wh_vs_that_with_gap",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "wh_vs_that_with_gap",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_wh_vs_that_with_gap_long_distance": {
+      "task": "blimp_wh_vs_that_with_gap_long_distance",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "wh_vs_that_with_gap_long_distance",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "blimp": "N/A",
+    "blimp_adjunct_island": 1.0,
+    "blimp_anaphor_gender_agreement": 1.0,
+    "blimp_anaphor_number_agreement": 1.0,
+    "blimp_animate_subject_passive": 1.0,
+    "blimp_animate_subject_trans": 1.0,
+    "blimp_causative": 1.0,
+    "blimp_complex_NP_island": 1.0,
+    "blimp_coordinate_structure_constraint_complex_left_branch": 1.0,
+    "blimp_coordinate_structure_constraint_object_extraction": 1.0,
+    "blimp_determiner_noun_agreement_1": 1.0,
+    "blimp_determiner_noun_agreement_2": 1.0,
+    "blimp_determiner_noun_agreement_irregular_1": 1.0,
+    "blimp_determiner_noun_agreement_irregular_2": 1.0,
+    "blimp_determiner_noun_agreement_with_adj_2": 1.0,
+    "blimp_determiner_noun_agreement_with_adj_irregular_1": 1.0,
+    "blimp_determiner_noun_agreement_with_adj_irregular_2": 1.0,
+    "blimp_determiner_noun_agreement_with_adjective_1": 1.0,
+    "blimp_distractor_agreement_relational_noun": 1.0,
+    "blimp_distractor_agreement_relative_clause": 1.0,
+    "blimp_drop_argument": 1.0,
+    "blimp_ellipsis_n_bar_1": 1.0,
+    "blimp_ellipsis_n_bar_2": 1.0,
+    "blimp_existential_there_object_raising": 1.0,
+    "blimp_existential_there_quantifiers_1": 1.0,
+    "blimp_existential_there_quantifiers_2": 1.0,
+    "blimp_existential_there_subject_raising": 1.0,
+    "blimp_expletive_it_object_raising": 1.0,
+    "blimp_inchoative": 1.0,
+    "blimp_intransitive": 1.0,
+    "blimp_irregular_past_participle_adjectives": 1.0,
+    "blimp_irregular_past_participle_verbs": 1.0,
+    "blimp_irregular_plural_subject_verb_agreement_1": 1.0,
+    "blimp_irregular_plural_subject_verb_agreement_2": 1.0,
+    "blimp_left_branch_island_echo_question": 1.0,
+    "blimp_left_branch_island_simple_question": 1.0,
+    "blimp_matrix_question_npi_licensor_present": 1.0,
+    "blimp_npi_present_1": 1.0,
+    "blimp_npi_present_2": 1.0,
+    "blimp_only_npi_licensor_present": 1.0,
+    "blimp_only_npi_scope": 1.0,
+    "blimp_passive_1": 1.0,
+    "blimp_passive_2": 1.0,
+    "blimp_principle_A_c_command": 1.0,
+    "blimp_principle_A_case_1": 1.0,
+    "blimp_principle_A_case_2": 1.0,
+    "blimp_principle_A_domain_1": 1.0,
+    "blimp_principle_A_domain_2": 1.0,
+    "blimp_principle_A_domain_3": 1.0,
+    "blimp_principle_A_reconstruction": 1.0,
+    "blimp_regular_plural_subject_verb_agreement_1": 1.0,
+    "blimp_regular_plural_subject_verb_agreement_2": 1.0,
+    "blimp_sentential_negation_npi_licensor_present": 1.0,
+    "blimp_sentential_negation_npi_scope": 1.0,
+    "blimp_sentential_subject_island": 1.0,
+    "blimp_superlative_quantifiers_1": 1.0,
+    "blimp_superlative_quantifiers_2": 1.0,
+    "blimp_tough_vs_raising_1": 1.0,
+    "blimp_tough_vs_raising_2": 1.0,
+    "blimp_transitive": 1.0,
+    "blimp_wh_island": 1.0,
+    "blimp_wh_questions_object_gap": 1.0,
+    "blimp_wh_questions_subject_gap": 1.0,
+    "blimp_wh_questions_subject_gap_long_distance": 1.0,
+    "blimp_wh_vs_that_no_gap": 1.0,
+    "blimp_wh_vs_that_no_gap_long_distance": 1.0,
+    "blimp_wh_vs_that_with_gap": 1.0,
+    "blimp_wh_vs_that_with_gap_long_distance": 1.0
+  },
+  "n-shot": {
+    "blimp": 0,
+    "blimp_adjunct_island": 0,
+    "blimp_anaphor_gender_agreement": 0,
+    "blimp_anaphor_number_agreement": 0,
+    "blimp_animate_subject_passive": 0,
+    "blimp_animate_subject_trans": 0,
+    "blimp_causative": 0,
+    "blimp_complex_NP_island": 0,
+    "blimp_coordinate_structure_constraint_complex_left_branch": 0,
+    "blimp_coordinate_structure_constraint_object_extraction": 0,
+    "blimp_determiner_noun_agreement_1": 0,
+    "blimp_determiner_noun_agreement_2": 0,
+    "blimp_determiner_noun_agreement_irregular_1": 0,
+    "blimp_determiner_noun_agreement_irregular_2": 0,
+    "blimp_determiner_noun_agreement_with_adj_2": 0,
+    "blimp_determiner_noun_agreement_with_adj_irregular_1": 0,
+    "blimp_determiner_noun_agreement_with_adj_irregular_2": 0,
+    "blimp_determiner_noun_agreement_with_adjective_1": 0,
+    "blimp_distractor_agreement_relational_noun": 0,
+    "blimp_distractor_agreement_relative_clause": 0,
+    "blimp_drop_argument": 0,
+    "blimp_ellipsis_n_bar_1": 0,
+    "blimp_ellipsis_n_bar_2": 0,
+    "blimp_existential_there_object_raising": 0,
+    "blimp_existential_there_quantifiers_1": 0,
+    "blimp_existential_there_quantifiers_2": 0,
+    "blimp_existential_there_subject_raising": 0,
+    "blimp_expletive_it_object_raising": 0,
+    "blimp_inchoative": 0,
+    "blimp_intransitive": 0,
+    "blimp_irregular_past_participle_adjectives": 0,
+    "blimp_irregular_past_participle_verbs": 0,
+    "blimp_irregular_plural_subject_verb_agreement_1": 0,
+    "blimp_irregular_plural_subject_verb_agreement_2": 0,
+    "blimp_left_branch_island_echo_question": 0,
+    "blimp_left_branch_island_simple_question": 0,
+    "blimp_matrix_question_npi_licensor_present": 0,
+    "blimp_npi_present_1": 0,
+    "blimp_npi_present_2": 0,
+    "blimp_only_npi_licensor_present": 0,
+    "blimp_only_npi_scope": 0,
+    "blimp_passive_1": 0,
+    "blimp_passive_2": 0,
+    "blimp_principle_A_c_command": 0,
+    "blimp_principle_A_case_1": 0,
+    "blimp_principle_A_case_2": 0,
+    "blimp_principle_A_domain_1": 0,
+    "blimp_principle_A_domain_2": 0,
+    "blimp_principle_A_domain_3": 0,
+    "blimp_principle_A_reconstruction": 0,
+    "blimp_regular_plural_subject_verb_agreement_1": 0,
+    "blimp_regular_plural_subject_verb_agreement_2": 0,
+    "blimp_sentential_negation_npi_licensor_present": 0,
+    "blimp_sentential_negation_npi_scope": 0,
+    "blimp_sentential_subject_island": 0,
+    "blimp_superlative_quantifiers_1": 0,
+    "blimp_superlative_quantifiers_2": 0,
+    "blimp_tough_vs_raising_1": 0,
+    "blimp_tough_vs_raising_2": 0,
+    "blimp_transitive": 0,
+    "blimp_wh_island": 0,
+    "blimp_wh_questions_object_gap": 0,
+    "blimp_wh_questions_subject_gap": 0,
+    "blimp_wh_questions_subject_gap_long_distance": 0,
+    "blimp_wh_vs_that_no_gap": 0,
+    "blimp_wh_vs_that_no_gap_long_distance": 0,
+    "blimp_wh_vs_that_with_gap": 0,
+    "blimp_wh_vs_that_with_gap_long_distance": 0
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=/home/bastian/Dokumente/baby_llamas/models/final_20",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": "cuda",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null
+  },
+  "git_hash": null
+}

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<pad>"
+}

tokenizer_config.json

@@ -0,0 +1,8 @@
+{
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4ed56affee867809a1f3c1a26f8cc48e4c93e78e6accaf0951a246d171a4a29
+size 4536