Upload tokenizer

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<|endoftext|>": 437
+}

merges.txt

@@ -0,0 +1,177 @@
+#version: 0.2
+Þ ¦
+Ġ Þ
+Þ °
+Þ ¬
+Þ ª
+Þ Ĥ
+Þ ¨
+Þ ĩ
+Þ §
+Þ ĥ
+Þ ī
+Þ Ĩ
+Þ Ī
+ĠÞ ĩ
+Þ İ
+Þ ©
+Þ Į
+Þ Ģ
+Þ ģ
+Þ ĭ
+Þ IJ
+Þ į
+Þ Ń
+Þ ®
+Þ Ĭ
+ĠÞ Ģ
+ĠÞ ī
+ĠÞ Ĩ
+Þ Ħ
+ĠÞ Ħ
+ĠÞ Ī
+ĠÞ ĭ
+ĠÞ Ĭ
+ĠÞ Ĥ
+Þ ħ
+Þ Ķ
+Þ ij
+Þ ĵ
+ĠÞ İ
+Þ ¯
+Þ «
+Þ ĸ
+ĠÞ Į
+ĠÞ ĥ
+ĠÞ IJ
+ĠÞ į
+Þ Ĵ
+ÞĤ Þİ
+Þ Ŀ
+ÞĤ Þij
+Þ ķ
+ĠÞ ĸ
+Þ Ĺ
+Ø Ł
+ÞĤ Þĭ
+ĠÞ Ĵ
+Þ ¢
+Þ ¤
+Þ Ļ
+ĠÞ Ļ
+ĠÞ ¢
+ĠÞ Ŀ
+Þ¦ ØŁ
+ĠÞ ļ
+ĠÞ ĵ
+Þ ŀ
+ĠÞ Ķ
+ĠÞ ij
+ĠÞ ķ
+ĠÞ ¤
+Þ ļ
+ÞĤ ÞĦ
+ï ·
+Þ¬ ØŁ
+Þ ı
+ï· ²
+Þ ł
+Þ° ØŁ
+ĠÞ ŀ
+Ø Į
+Þ Ł
+Þ° .
+Þ¬ !
+Þ ĺ
+Ġ ï·²
+Þ© .
+Þĥ ÞĪ
+Þĥ Þĵ
+ĠÞ Ĺ
+Þ© ØŁ
+Þ§ ØŁ
+Þ £
+Þ¦ Þ¦
+â Ģ
+ĠÞ ħ
+Þ¬ Þ¬
+ÞĤ Þĩ
+Þ¨ .
+Þĥ ÞĨ
+Þĥ Þİ
+Þ¯ ØŁ
+Ġ -
+Ġ ï·
+Ġï· º
+Þ Ľ
+ĠÞ Ł
+ĠÞ ł
+Þ° Þ°
+Þ° ØĮ
+Þª Þª
+Þª ØŁ
+Þ¨ ØĮ
+Þĥ ÞĮ
+Þĥ Þij
+Þĥ Þķ
+ÞŃ !
+âĢ ĺ
+Þª ØĮ
+Ġ âĢĺ
+Þ¦ Þ¨
+ĠÞ ĺ
+Þ¬ .
+Þ¬ Þ°
+Þĩ Þĩ
+Þ§ Þ§
+Þĥ ÞĤ
+Þĥ ÞĦ
+Þİ Þĩ
+âĢ Ļ
+Þ ¥
+Þĥ Þī
+ĠÞ £
+Þ ¡
+Ġ ;
+Ġ ØŁ
+Þ¦ !
+Þ¦ Þª
+Þ¦ Þ§
+Þ° !
+Þ° âĢĻ
+Þª Þ¦
+ÞĤ ÞĤ
+ÞĤ ÞĨ
+ÞĤ ÞĢ
+ÞĤ Þģ
+ÞĤ Þĸ
+Þ¨ -
+Þ¨ Þ©
+Þĩ Þĥ
+Þ§ Þ°
+Þ§ ØĮ
+Þĥ ÞIJ
+Þĥ Þį
+Þĥ ÞĬ
+Þĥ ÞĴ
+Þĥ ÞĹ
+Þī ÞĪ
+ÞĪ Þĩ
+Þİ Þĥ
+Þİ Þİ
+Þ© :
+Þ© ;
+Þ© ØĮ
+Þ© âĢĻ
+ÞĢ Þĩ
+ÞĢ Þĭ
+ÞŃ .
+ÞŃ ØŁ
+ÞŃ ØĮ
+Þ® Þ¦
+ĠÞĤ Þij
+Þ¯ !
+Þ¯ .
+ĠÞIJ Þĵ
+Þı Þģ
+ï·² Þİ

special_tokens_map.json

@@ -1,8 +1,22 @@
 {
-  "pad_token": {
-    "content": "[PAD]",
+  "bos_token": {
+    "content": "<|endoftext|>",
     "lstrip": false,
-    "normalized": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
     "rstrip": false,
     "single_word": false
   }

tokenizer_config.json

@@ -1,49 +1,23 @@
 {
+  "add_bos_token": false,
+  "add_prefix_space": false,
   "added_tokens_decoder": {
-    "0": {
-      "content": "<pad>",
+    "437": {
+      "content": "<|endoftext|>",
       "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "1": {
-      "content": "<s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "2": {
-      "content": "</s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "3": {
-      "content": "<unk>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "8000": {
-      "content": "[PAD]",
-      "lstrip": false,
-      "normalized": false,
+      "normalized": true,
       "rstrip": false,
       "single_word": false,
       "special": true
     }
   },
+  "bos_token": "<|endoftext|>",
   "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
   "extra_special_tokens": {},
   "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "[PAD]",
-  "tokenizer_class": "PreTrainedTokenizerFast"
+  "pad_token": null,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
 }

vocab.json

@@ -0,0 +1 @@
+{"<s>":0,"<pad>":1,"</s>":2,"<unk>":3,"<mask>":4,"!":5,"\"":6,"#":7,"$":8,"%":9,"&":10,"'":11,"(":12,")":13,"*":14,"+":15,",":16,"-":17,".":18,"/":19,"0":20,"1":21,"2":22,"3":23,"4":24,"5":25,"6":26,"7":27,"8":28,"9":29,":":30,";":31,"<":32,"=":33,">":34,"?":35,"@":36,"A":37,"B":38,"C":39,"D":40,"E":41,"F":42,"G":43,"H":44,"I":45,"J":46,"K":47,"L":48,"M":49,"N":50,"O":51,"P":52,"Q":53,"R":54,"S":55,"T":56,"U":57,"V":58,"W":59,"X":60,"Y":61,"Z":62,"[":63,"\\":64,"]":65,"^":66,"_":67,"`":68,"a":69,"b":70,"c":71,"d":72,"e":73,"f":74,"g":75,"h":76,"i":77,"j":78,"k":79,"l":80,"m":81,"n":82,"o":83,"p":84,"q":85,"r":86,"s":87,"t":88,"u":89,"v":90,"w":91,"x":92,"y":93,"z":94,"{":95,"|":96,"}":97,"~":98,"¡":99,"¢":100,"£":101,"¤":102,"¥":103,"¦":104,"§":105,"¨":106,"©":107,"ª":108,"«":109,"¬":110,"®":111,"¯":112,"°":113,"±":114,"²":115,"³":116,"´":117,"µ":118,"¶":119,"·":120,"¸":121,"¹":122,"º":123,"»":124,"¼":125,"½":126,"¾":127,"¿":128,"À":129,"Á":130,"Â":131,"Ã":132,"Ä":133,"Å":134,"Æ":135,"Ç":136,"È":137,"É":138,"Ê":139,"Ë":140,"Ì":141,"Í":142,"Î":143,"Ï":144,"Ð":145,"Ñ":146,"Ò":147,"Ó":148,"Ô":149,"Õ":150,"Ö":151,"×":152,"Ø":153,"Ù":154,"Ú":155,"Û":156,"Ü":157,"Ý":158,"Þ":159,"ß":160,"à":161,"á":162,"â":163,"ã":164,"ä":165,"å":166,"æ":167,"ç":168,"è":169,"é":170,"ê":171,"ë":172,"ì":173,"í":174,"î":175,"ï":176,"ð":177,"ñ":178,"ò":179,"ó":180,"ô":181,"õ":182,"ö":183,"÷":184,"ø":185,"ù":186,"ú":187,"û":188,"ü":189,"ý":190,"þ":191,"ÿ":192,"Ā":193,"ā":194,"Ă":195,"ă":196,"Ą":197,"ą":198,"Ć":199,"ć":200,"Ĉ":201,"ĉ":202,"Ċ":203,"ċ":204,"Č":205,"č":206,"Ď":207,"ď":208,"Đ":209,"đ":210,"Ē":211,"ē":212,"Ĕ":213,"ĕ":214,"Ė":215,"ė":216,"Ę":217,"ę":218,"Ě":219,"ě":220,"Ĝ":221,"ĝ":222,"Ğ":223,"ğ":224,"Ġ":225,"ġ":226,"Ģ":227,"ģ":228,"Ĥ":229,"ĥ":230,"Ħ":231,"ħ":232,"Ĩ":233,"ĩ":234,"Ī":235,"ī":236,"Ĭ":237,"ĭ":238,"Į":239,"į":240,"İ":241,"ı":242,"IJ":243,"ij":244,"Ĵ":245,"ĵ":246,"Ķ":247,"ķ":248,"ĸ":249,"Ĺ":250,"ĺ":251,"Ļ":252,"ļ":253,"Ľ":254,"ľ":255,"Ŀ":256,"ŀ":257,"Ł":258,"ł":259,"Ń":260,"Þ¦":261,"ĠÞ":262,"Þ°":263,"Þ¬":264,"Þª":265,"ÞĤ":266,"Þ¨":267,"Þĩ":268,"Þ§":269,"Þĥ":270,"Þī":271,"ÞĨ":272,"ÞĪ":273,"ĠÞĩ":274,"Þİ":275,"Þ©":276,"ÞĮ":277,"ÞĢ":278,"Þģ":279,"Þĭ":280,"ÞIJ":281,"Þį":282,"ÞŃ":283,"Þ®":284,"ÞĬ":285,"ĠÞĢ":286,"ĠÞī":287,"ĠÞĨ":288,"ÞĦ":289,"ĠÞĦ":290,"ĠÞĪ":291,"ĠÞĭ":292,"ĠÞĬ":293,"ĠÞĤ":294,"Þħ":295,"ÞĶ":296,"Þij":297,"Þĵ":298,"ĠÞİ":299,"Þ¯":300,"Þ«":301,"Þĸ":302,"ĠÞĮ":303,"ĠÞĥ":304,"ĠÞIJ":305,"ĠÞį":306,"ÞĴ":307,"ÞĤÞİ":308,"ÞĿ":309,"ÞĤÞij":310,"Þķ":311,"ĠÞĸ":312,"ÞĹ":313,"ØŁ":314,"ÞĤÞĭ":315,"ĠÞĴ":316,"Þ¢":317,"Þ¤":318,"ÞĻ":319,"ĠÞĻ":320,"ĠÞ¢":321,"ĠÞĿ":322,"Þ¦ØŁ":323,"ĠÞļ":324,"ĠÞĵ":325,"Þŀ":326,"ĠÞĶ":327,"ĠÞij":328,"ĠÞķ":329,"ĠÞ¤":330,"Þļ":331,"ÞĤÞĦ":332,"ï·":333,"Þ¬ØŁ":334,"Þı":335,"ï·²":336,"Þł":337,"Þ°ØŁ":338,"ĠÞŀ":339,"ØĮ":340,"ÞŁ":341,"Þ°.":342,"Þ¬!":343,"Þĺ":344,"Ġï·²":345,"Þ©.":346,"ÞĥÞĪ":347,"ÞĥÞĵ":348,"ĠÞĹ":349,"Þ©ØŁ":350,"Þ§ØŁ":351,"Þ£":352,"Þ¦Þ¦":353,"âĢ":354,"ĠÞħ":355,"Þ¬Þ¬":356,"ÞĤÞĩ":357,"Þ¨.":358,"ÞĥÞĨ":359,"ÞĥÞİ":360,"Þ¯ØŁ":361,"Ġ-":362,"Ġï·":363,"Ġï·º":364,"ÞĽ":365,"ĠÞŁ":366,"ĠÞł":367,"Þ°Þ°":368,"Þ°ØĮ":369,"ÞªÞª":370,"ÞªØŁ":371,"Þ¨ØĮ":372,"ÞĥÞĮ":373,"ÞĥÞij":374,"ÞĥÞķ":375,"ÞŃ!":376,"âĢĺ":377,"ÞªØĮ":378,"ĠâĢĺ":379,"Þ¦Þ¨":380,"ĠÞĺ":381,"Þ¬.":382,"Þ¬Þ°":383,"ÞĩÞĩ":384,"Þ§Þ§":385,"ÞĥÞĤ":386,"ÞĥÞĦ":387,"ÞİÞĩ":388,"âĢĻ":389,"Þ¥":390,"ÞĥÞī":391,"ĠÞ£":392,"Þ¡":393,"Ġ;":394,"ĠØŁ":395,"Þ¦!":396,"Þ¦Þª":397,"Þ¦Þ§":398,"Þ°!":399,"Þ°âĢĻ":400,"ÞªÞ¦":401,"ÞĤÞĤ":402,"ÞĤÞĨ":403,"ÞĤÞĢ":404,"ÞĤÞģ":405,"ÞĤÞĸ":406,"Þ¨-":407,"Þ¨Þ©":408,"ÞĩÞĥ":409,"Þ§Þ°":410,"Þ§ØĮ":411,"ÞĥÞIJ":412,"ÞĥÞį":413,"ÞĥÞĬ":414,"ÞĥÞĴ":415,"ÞĥÞĹ":416,"ÞīÞĪ":417,"ÞĪÞĩ":418,"ÞİÞĥ":419,"ÞİÞİ":420,"Þ©:":421,"Þ©;":422,"Þ©ØĮ":423,"Þ©âĢĻ":424,"ÞĢÞĩ":425,"ÞĢÞĭ":426,"ÞŃ.":427,"ÞŃØŁ":428,"ÞŃØĮ":429,"Þ®Þ¦":430,"ĠÞĤÞij":431,"Þ¯!":432,"Þ¯.":433,"ĠÞIJÞĵ":434,"ÞıÞģ":435,"ï·²Þİ":436}