fix autotokenizer

tokenizer.json

@@ -37,7 +37,7 @@
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
-      "special": false
+      "special": true
     },
     {
       "id": 4,
@@ -46,7 +46,7 @@
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
-      "special": false
+      "special": true
     },
     {
       "id": 5,
@@ -55,7 +55,7 @@
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
-      "special": false
+      "special": true
     },
     {
       "id": 6,
@@ -64,15 +64,26 @@
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
-      "special": false
+      "special": true
     }
   ],
-  "normalizer": null,
+  "normalizer": {
+    "type": "Replace",
+    "pattern": {
+      "Regex": " {2,}"
+    },
+    "content": "▁"
+  },
   "pre_tokenizer": {
-    "type": "Metaspace",
-    "replacement": "▁",
-    "prepend_scheme": "first",
-    "split": false
+    "type": "Sequence",
+    "pretokenizers": [
+      {
+        "type": "Metaspace",
+        "replacement": "▁",
+        "prepend_scheme": "first",
+        "split": false
+      }
+    ]
   },
   "post_processor": {
     "type": "TemplateProcessing",

tokenizer_config.json

@@ -33,7 +33,7 @@
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "4": {
       "content": "[/INST]",
@@ -41,7 +41,7 @@
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "5": {
       "content": "<<SYS>>",
@@ -49,7 +49,7 @@
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "6": {
       "content": "<</SYS>>",
@@ -57,7 +57,7 @@
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     }
   },
   "bos_token": "<s>",
@@ -69,7 +69,7 @@
   "pad_token": null,
   "sp_model_kwargs": {},
   "spaces_between_special_tokens": false,
-  "tokenizer_class": "LlamaTokenizer",
+  "tokenizer_class": "PreTrainedTokenizerFast",
   "unk_token": "<unk>",
   "use_default_system_prompt": false
 }