p1atdev
/

dart-v3-tokenizer-240912

Inference Endpoints

Model card Files Files and versions Community

p1atdev commited on Sep 11

Commit

ca99846

•

1 Parent(s): d0fdf2d

Upload tokenizer

Files changed (1) hide show

tokenizer.json +1 -1

tokenizer.json CHANGED Viewed

@@ -1124,7 +1124,7 @@
   "pre_tokenizer": {
     "type": "Split",
     "pattern": {
-      "Regex": "(?:\\s)*"
     },
     "behavior": "Removed",
     "invert": false

   "pre_tokenizer": {
     "type": "Split",
     "pattern": {
+      "Regex": ",(?:\\s)*"
     },
     "behavior": "Removed",
     "invert": false