Hugging Face's logo

stephantulkens
/
NIFE-gte-modernbert-base

Safetensors
English
Model card Files
xet
Community
1
stephantulkens commited on
Commit
53f5646
·
unverified ·
1 Parent(s): 09460c0

fix tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer.json +1 -75
tokenizer.json CHANGED
@@ -50,81 +50,7 @@
50
  "pre_tokenizer": {
51
  "type": "BertPreTokenizer"
52
  },
53
- "post_processor": {
54
- "type": "TemplateProcessing",
55
- "single": [
56
- {
57
- "SpecialToken": {
58
- "id": "[CLS]",
59
- "type_id": 0
60
- }
61
- },
62
- {
63
- "Sequence": {
64
- "id": "A",
65
- "type_id": 0
66
- }
67
- },
68
- {
69
- "SpecialToken": {
70
- "id": "[SEP]",
71
- "type_id": 0
72
- }
73
- }
74
- ],
75
- "pair": [
76
- {
77
- "SpecialToken": {
78
- "id": "[CLS]",
79
- "type_id": 0
80
- }
81
- },
82
- {
83
- "Sequence": {
84
- "id": "A",
85
- "type_id": 0
86
- }
87
- },
88
- {
89
- "SpecialToken": {
90
- "id": "[SEP]",
91
- "type_id": 0
92
- }
93
- },
94
- {
95
- "Sequence": {
96
- "id": "B",
97
- "type_id": 1
98
- }
99
- },
100
- {
101
- "SpecialToken": {
102
- "id": "[SEP]",
103
- "type_id": 1
104
- }
105
- }
106
- ],
107
- "special_tokens": {
108
- "[CLS]": {
109
- "id": "[CLS]",
110
- "ids": [
111
- 101
112
- ],
113
- "tokens": [
114
- "[CLS]"
115
- ]
116
- },
117
- "[SEP]": {
118
- "id": "[SEP]",
119
- "ids": [
120
- 102
121
- ],
122
- "tokens": [
123
- "[SEP]"
124
- ]
125
- }
126
- }
127
- },
128
  "decoder": {
129
  "type": "WordPiece",
130
  "prefix": "##",
 
50
  "pre_tokenizer": {
51
  "type": "BertPreTokenizer"
52
  },
53
+ "post_processor": null,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
54
  "decoder": {
55
  "type": "WordPiece",
56
  "prefix": "##",