TheBloke commited on
Commit
9cbf002
β€’
1 Parent(s): 05b1599

Fix tokenizer.json

Browse files
Files changed (1) hide show
  1. tokenizer.json +1 -10
tokenizer.json CHANGED
@@ -32,15 +32,6 @@
32
  },
33
  {
34
  "id": 32000,
35
- "content": "<pad>",
36
- "single_word": false,
37
- "lstrip": false,
38
- "rstrip": false,
39
- "normalized": true,
40
- "special": false
41
- },
42
- {
43
- "id": 32001,
44
  "content": "[PAD]",
45
  "single_word": false,
46
  "lstrip": false,
@@ -93406,4 +93397,4 @@
93406
  "▁ ▁▁▁▁▁▁▁▁▁▁▁▁▁▁"
93407
  ]
93408
  }
93409
- }
 
32
  },
33
  {
34
  "id": 32000,
 
 
 
 
 
 
 
 
 
35
  "content": "[PAD]",
36
  "single_word": false,
37
  "lstrip": false,
 
93397
  "▁ ▁▁▁▁▁▁▁▁▁▁▁▁▁▁"
93398
  ]
93399
  }
93400
+ }