Hugging Face's logo

jordiclive
/
test-tokenizer

t5
Model card Files Community
test-tokenizer
File size: 3,168 Bytes 
e564a6b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
 
{
  "<100>": 32231,
  "<101>": 32227,
  "<102>": 32182,
  "<103>": 32156,
  "<104>": 32163,
  "<105>": 32172,
  "<106>": 32177,
  "<107>": 32233,
  "<108>": 32223,
  "<110>": 32129,
  "<113>": 32154,
  "<116>": 32183,
  "<11>": 32246,
  "<124>": 32192,
  "<126>": 32139,
  "<128>": 32232,
  "<12>": 32203,
  "<131>": 32256,
  "<132>": 32152,
  "<133>": 32188,
  "<13>": 32118,
  "<14>": 32175,
  "<151>": 32217,
  "<152>": 32142,
  "<153>": 32116,
  "<156>": 32237,
  "<15>": 32235,
  "<162>": 32158,
  "<166>": 32208,
  "<16>": 32136,
  "<175>": 32190,
  "<176>": 32187,
  "<177>": 32247,
  "<178>": 32140,
  "<179>": 32185,
  "<17>": 32251,
  "<180>": 32120,
  "<18>": 32130,
  "<191>": 32240,
  "<192>": 32220,
  "<1>": 32141,
  "<200>": 32161,
  "<201>": 32164,
  "<204>": 32193,
  "<207>": 32119,
  "<208>": 32195,
  "<209>": 32228,
  "<20>": 32201,
  "<210>": 32209,
  "<211>": 32166,
  "<217>": 32248,
  "<218>": 32249,
  "<21>": 32213,
  "<220>": 32114,
  "<221>": 32243,
  "<228>": 32155,
  "<22>": 32212,
  "<237>": 32131,
  "<23>": 32245,
  "<240>": 32204,
  "<241>": 32261,
  "<242>": 32215,
  "<245>": 32121,
  "<249>": 32123,
  "<24>": 32167,
  "<256>": 32134,
  "<257>": 32258,
  "<25>": 32214,
  "<260>": 32186,
  "<267>": 32128,
  "<270>": 32179,
  "<272>": 32216,
  "<273>": 32238,
  "<274>": 32148,
  "<275>": 32125,
  "<279>": 32219,
  "<284>": 32159,
  "<285>": 32230,
  "<286>": 32146,
  "<287>": 32150,
  "<289>": 32259,
  "<28>": 32117,
  "<290>": 32124,
  "<298>": 32165,
  "<299>": 32169,
  "<29>": 32218,
  "<2>": 32253,
  "<302>": 32143,
  "<304>": 32184,
  "<305>": 32180,
  "<306>": 32160,
  "<307>": 32221,
  "<309>": 32122,
  "<311>": 32222,
  "<312>": 32250,
  "<316>": 32197,
  "<317>": 32229,
  "<319>": 32200,
  "<31>": 32111,
  "<320>": 32257,
  "<322>": 32153,
  "<325>": 32173,
  "<327>": 32194,
  "<32>": 32133,
  "<330>": 32207,
  "<331>": 32138,
  "<332>": 32205,
  "<334>": 32149,
  "<335>": 32112,
  "<337>": 32162,
  "<338>": 32206,
  "<359>": 32260,
  "<361>": 32198,
  "<363>": 32168,
  "<3>": 32224,
  "<40>": 32244,
  "<414>": 32115,
  "<42>": 32170,
  "<46>": 32135,
  "<48>": 32181,
  "<49>": 32191,
  "<4>": 32147,
  "<50>": 32113,
  "<54>": 32202,
  "<56>": 32262,
  "<57>": 32174,
  "<58>": 32127,
  "<5>": 32151,
  "<62>": 32254,
  "<64>": 32132,
  "<6>": 32239,
  "<70>": 32234,
  "<72>": 32137,
  "<73>": 32196,
  "<78>": 32226,
  "<79>": 32145,
  "<7>": 32236,
  "<81>": 32176,
  "<82>": 32199,
  "<83>": 32210,
  "<84>": 32157,
  "<85>": 32252,
  "<86>": 32178,
  "<87>": 32241,
  "<88>": 32211,
  "<89>": 32225,
  "<8>": 32255,
  "<90>": 32144,
  "<91>": 32189,
  "<97>": 32171,
  "<99999>": 32242,
  "<99>": 32126,
  "<category_sep>": 32269,
  "<comment_sep>": 32267,
  "<example_sep>": 32263,
  "<ind_sep>": 32264,
  "<org_prompt_sep>": 32266,
  "<question_sep>": 32270,
  "<score_0>": 32100,
  "<score_10>": 32110,
  "<score_1>": 32101,
  "<score_2>": 32102,
  "<score_3>": 32103,
  "<score_4>": 32104,
  "<score_5>": 32105,
  "<score_6>": 32106,
  "<score_7>": 32107,
  "<score_8>": 32108,
  "<score_9>": 32109,
  "<sub_ind_sep>": 32265,
  "<theme_classification_token>": 32271,
  "<theme_sep>": 32268
}