Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

artifact.metadata

@@ -0,0 +1,59 @@
+{
+    "query_token_id": "[unused0]",
+    "doc_token_id": "[unused1]",
+    "query_token": "[Q]",
+    "doc_token": "[D]",
+    "ncells": null,
+    "centroid_score_threshold": null,
+    "ndocs": null,
+    "load_index_with_mmap": false,
+    "index_path": null,
+    "nbits": 2,
+    "kmeans_niters": 4,
+    "resume": false,
+    "similarity": "cosine",
+    "bsize": 128,
+    "accumsteps": 1,
+    "lr": 5e-6,
+    "maxsteps": 200000,
+    "save_every": null,
+    "warmup": 20000,
+    "warmup_bert": null,
+    "relu": false,
+    "nway": 2,
+    "use_ib_negatives": true,
+    "reranker": false,
+    "distillation_alpha": 1.0,
+    "ignore_scores": true,
+    "model_name": "facebook\/xmod-base",
+    "query_maxlen": 32,
+    "attend_to_mask_tokens": true,
+    "interaction": "colbert",
+    "dim": 128,
+    "doc_maxlen": 256,
+    "mask_punctuation": true,
+    "checkpoint": "facebook\/xmod-base",
+    "triples": "data\/mmarco\/tuples.train.scores-ids.2nway.25.6M.jsonl",
+    "collection": "data\/mmarco\/english_collection.tsv",
+    "queries": "data\/mmarco\/english_queries.train.tsv",
+    "index_name": null,
+    "overwrite": false,
+    "root": "output\/training\/mmarco",
+    "experiment": "colbert",
+    "index_root": null,
+    "name": "2024-01-24_18.33-facebook-xmod-base-mmarco-en",
+    "rank": 0,
+    "nranks": 1,
+    "amp": true,
+    "gpus": 1,
+    "avoid_fork_if_possible": false,
+    "meta": {
+        "hostname": "jupyterlab-gpu-2-4-tljq4",
+        "git_branch": "main",
+        "git_hash": "2760099557eff9c5ca8d58bd23198a71161eaacf",
+        "git_commit_datetime": "2024-01-24 17:11:38+01:00",
+        "current_datetime": "Jan 25, 2024 ;  2:08AM CET (+0100)",
+        "cmd": "src\/training\/colbertv2.py --dataset mmarco --language en --do_train --model_name facebook\/xmod-base --dim 128 --similarity cosine --doc_maxlen 256 --query_maxlen 32 --mask_punctuation --attend_to_mask_tokens --maxsteps 200000 --warmup 20000 --lr 5e-6 --bsize 128 --accumsteps 1 --nway 2 --use_ib_negatives --ignore_scores --distillation_alpha 1.0 --nbits 2 --kmeans_niters 4 --data_dir data --output_dir output\/training",
+        "version": "colbert-v0.4"
+    }
+}

config.json

@@ -0,0 +1,116 @@
+{
+  "_name_or_path": "facebook/xmod-base",
+  "adapter_layer_norm": false,
+  "adapter_reduction_factor": 2,
+  "adapter_reuse_layer_norm": true,
+  "architectures": [
+    "HF_ColBERT"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "default_language": "en_XX",
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "languages": [
+    "en_XX",
+    "id_ID",
+    "vi_VN",
+    "ru_RU",
+    "fa_IR",
+    "sv_SE",
+    "ja_XX",
+    "fr_XX",
+    "de_DE",
+    "ro_RO",
+    "ko_KR",
+    "hu_HU",
+    "es_XX",
+    "fi_FI",
+    "uk_UA",
+    "da_DK",
+    "pt_XX",
+    "no_XX",
+    "th_TH",
+    "pl_PL",
+    "bg_BG",
+    "nl_XX",
+    "zh_CN",
+    "he_IL",
+    "el_GR",
+    "it_IT",
+    "sk_SK",
+    "hr_HR",
+    "tr_TR",
+    "ar_AR",
+    "cs_CZ",
+    "lt_LT",
+    "hi_IN",
+    "zh_TW",
+    "ca_ES",
+    "ms_MY",
+    "sl_SI",
+    "lv_LV",
+    "ta_IN",
+    "bn_IN",
+    "et_EE",
+    "az_AZ",
+    "sq_AL",
+    "sr_RS",
+    "kk_KZ",
+    "ka_GE",
+    "tl_XX",
+    "ur_PK",
+    "is_IS",
+    "hy_AM",
+    "ml_IN",
+    "mk_MK",
+    "be_BY",
+    "la_VA",
+    "te_IN",
+    "eu_ES",
+    "gl_ES",
+    "mn_MN",
+    "kn_IN",
+    "ne_NP",
+    "sw_KE",
+    "si_LK",
+    "mr_IN",
+    "af_ZA",
+    "gu_IN",
+    "cy_GB",
+    "eo_EO",
+    "km_KH",
+    "ky_KG",
+    "uz_UZ",
+    "ps_AF",
+    "pa_IN",
+    "ga_IE",
+    "ha_NG",
+    "am_ET",
+    "lo_LA",
+    "ku_TR",
+    "so_SO",
+    "my_MM",
+    "or_IN",
+    "sa_IN"
+  ],
+  "layer_norm_eps": 1e-05,
+  "ln_before_adapter": true,
+  "max_position_embeddings": 514,
+  "model_type": "xmod",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "pre_norm": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.36.2",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 250002
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c74a0fabc96993d671aa8550eec103bd31ec62a5c50c570b41ec628a8985b3c
+size 3410847320

special_tokens_map.json

@@ -0,0 +1,15 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f59925fcb90c92b894cb93e51bb9b4a6105c5c249fe54ce1c704420ac39b81af
+size 17082756

tokenizer_config.json

@@ -0,0 +1,54 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250001": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "tokenizer_class": "XLMRobertaTokenizer",
+  "unk_token": "<unk>"
+}