End of training glittering-fireworks-202

merges.txt

@@ -1,4 +1,4 @@
-#version: 0.2
+#version: 0.2 - Trained by `huggingface/tokenizers`
 Ġ t
 Ġ a
 h e

special_tokens_map.json

@@ -2,7 +2,13 @@
   "bos_token": "<s>",
   "cls_token": "<s>",
   "eos_token": "</s>",
-  "mask_token": "<mask>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
   "pad_token": "<pad>",
   "sep_token": "</s>",
   "unk_token": "<unk>"

tokenizer_config.json

@@ -51,8 +51,9 @@
     "rstrip": false,
     "single_word": false
   },
-  "special_tokens_map_file": "/Users/martin/.cache/huggingface/hub/models--microsoft--codebert-base-mlm/snapshots/71b41964e82666ecdb8c72b11d8a02395e6fcda5/special_tokens_map.json",
+  "special_tokens_map_file": "/home/s2498103/.cache/huggingface/hub/models--microsoft--codebert-base-mlm/snapshots/71b41964e82666ecdb8c72b11d8a02395e6fcda5/special_tokens_map.json",
   "tokenizer_class": "RobertaTokenizer",
+  "trim_offsets": true,
   "unk_token": {
     "__type": "AddedToken",
     "content": "<unk>",
@@ -60,5 +61,6 @@
     "normalized": true,
     "rstrip": false,
     "single_word": false
-  }
+  },
+  "use_fast": true
 }