kkuramitsu commited on
Commit
6fb8f3b
1 Parent(s): dea97e2

Upload 5 files

Browse files
Files changed (5) hide show
  1. head.jsonl +397 -0
  2. new.jsonl +0 -0
  3. special_tokens_map.json +5 -0
  4. spiece.model +3 -0
  5. tokenizer_config.json +13 -0
head.jsonl ADDED
@@ -0,0 +1,397 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"id": 0, "type": 3, "piece": "<pad>", "score": 0.0}
2
+ {"id": 1, "type": 3, "piece": "</s>", "score": 0.0}
3
+ {"id": 2, "type": 3, "piece": "<s>", "score": 0.0}
4
+ {"id": 3, "type": 2, "piece": "<unk>", "score": 0.0}
5
+ {"id": 4, "type": 4, "piece": "<nl>", "score": 0.0}
6
+ {"id": 5, "type": 4, "piece": "<code>", "score": 0.0}
7
+ {"id": 6, "type": 4, "piece": "</code>", "score": 0.0}
8
+ {"id": 7, "type": 4, "piece": "```", "score": 0.0}
9
+ {"id": 8, "type": 4, "piece": "\"\"\"", "score": 0.0}
10
+ {"id": 9, "type": 4, "piece": "<mask>", "score": 0.0}
11
+ {"id": 10, "type": 4, "piece": "<sep>", "score": 0.0}
12
+ {"id": 11, "type": 4, "piece": "<C>", "score": 0.0}
13
+ {"id": 12, "type": 4, "piece": "▁<C>", "score": 0.0}
14
+ {"id": 13, "type": 4, "piece": "<K>", "score": 0.0}
15
+ {"id": 14, "type": 4, "piece": "<H>", "score": 0.0}
16
+ {"id": 15, "type": 4, "piece": "<X>", "score": 0.0}
17
+ {"id": 16, "type": 4, "piece": "<Y>", "score": 0.0}
18
+ {"id": 10, "type": 4, "piece": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", "score": 0.0}
19
+ {"id": 11, "type": 4, "piece": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", "score": 0.0}
20
+ {"id": 12, "type": 4, "piece": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", "score": 0.0}
21
+ {"id": 13, "type": 4, "piece": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", "score": 0.0}
22
+ {"id": 14, "type": 4, "piece": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", "score": 0.0}
23
+ {"id": 15, "type": 4, "piece": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", "score": 0.0}
24
+ {"id": 16, "type": 4, "piece": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", "score": 0.0}
25
+ {"id": 17, "type": 4, "piece": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", "score": 0.0}
26
+ {"id": 18, "type": 4, "piece": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", "score": 0.0}
27
+ {"id": 19, "type": 4, "piece": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", "score": 0.0}
28
+ {"id": 20, "type": 4, "piece": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁", "score": 0.0}
29
+ {"id": 21, "type": 4, "piece": "▁▁▁▁▁▁▁▁▁▁▁▁▁", "score": 0.0}
30
+ {"id": 22, "type": 4, "piece": "▁▁▁▁▁▁▁▁▁▁▁▁", "score": 0.0}
31
+ {"id": 23, "type": 4, "piece": "▁▁▁▁▁▁▁▁▁▁▁", "score": 0.0}
32
+ {"id": 24, "type": 4, "piece": "▁▁▁▁▁▁▁▁▁▁", "score": 0.0}
33
+ {"id": 25, "type": 4, "piece": "▁▁▁▁▁▁▁▁▁", "score": 0.0}
34
+ {"id": 26, "type": 4, "piece": "▁▁▁▁▁▁▁▁", "score": 0.0}
35
+ {"id": 27, "type": 4, "piece": "▁▁▁▁▁▁▁", "score": 0.0}
36
+ {"id": 28, "type": 4, "piece": "▁▁▁▁▁▁", "score": 0.0}
37
+ {"id": 29, "type": 4, "piece": "▁▁▁▁▁", "score": 0.0}
38
+ {"id": 30, "type": 4, "piece": "▁▁▁▁", "score": 0.0}
39
+ {"id": 31, "type": 4, "piece": "▁▁▁", "score": 0.0}
40
+ {"id": 32, "type": 4, "piece": "▁▁", "score": 0.0}
41
+ {"id": 33, "type": 4, "piece": "▁", "score": 0.0}
42
+ {"id": 17, "type": 6, "piece": "<0x00>", "score": 0.0}
43
+ {"id": 18, "type": 6, "piece": "<0x01>", "score": 0.0}
44
+ {"id": 19, "type": 6, "piece": "<0x02>", "score": 0.0}
45
+ {"id": 20, "type": 6, "piece": "<0x03>", "score": 0.0}
46
+ {"id": 21, "type": 6, "piece": "<0x04>", "score": 0.0}
47
+ {"id": 22, "type": 6, "piece": "<0x05>", "score": 0.0}
48
+ {"id": 23, "type": 6, "piece": "<0x06>", "score": 0.0}
49
+ {"id": 24, "type": 6, "piece": "<0x07>", "score": 0.0}
50
+ {"id": 25, "type": 6, "piece": "<0x08>", "score": 0.0}
51
+ {"id": 26, "type": 6, "piece": "<0x09>", "score": 0.0}
52
+ {"id": 27, "type": 6, "piece": "<0x0A>", "score": 0.0}
53
+ {"id": 28, "type": 6, "piece": "<0x0B>", "score": 0.0}
54
+ {"id": 29, "type": 6, "piece": "<0x0C>", "score": 0.0}
55
+ {"id": 30, "type": 6, "piece": "<0x0D>", "score": 0.0}
56
+ {"id": 31, "type": 6, "piece": "<0x0E>", "score": 0.0}
57
+ {"id": 32, "type": 6, "piece": "<0x0F>", "score": 0.0}
58
+ {"id": 33, "type": 6, "piece": "<0x10>", "score": 0.0}
59
+ {"id": 34, "type": 6, "piece": "<0x11>", "score": 0.0}
60
+ {"id": 35, "type": 6, "piece": "<0x12>", "score": 0.0}
61
+ {"id": 36, "type": 6, "piece": "<0x13>", "score": 0.0}
62
+ {"id": 37, "type": 6, "piece": "<0x14>", "score": 0.0}
63
+ {"id": 38, "type": 6, "piece": "<0x15>", "score": 0.0}
64
+ {"id": 39, "type": 6, "piece": "<0x16>", "score": 0.0}
65
+ {"id": 40, "type": 6, "piece": "<0x17>", "score": 0.0}
66
+ {"id": 41, "type": 6, "piece": "<0x18>", "score": 0.0}
67
+ {"id": 42, "type": 6, "piece": "<0x19>", "score": 0.0}
68
+ {"id": 43, "type": 6, "piece": "<0x1A>", "score": 0.0}
69
+ {"id": 44, "type": 6, "piece": "<0x1B>", "score": 0.0}
70
+ {"id": 45, "type": 6, "piece": "<0x1C>", "score": 0.0}
71
+ {"id": 46, "type": 6, "piece": "<0x1D>", "score": 0.0}
72
+ {"id": 47, "type": 6, "piece": "<0x1E>", "score": 0.0}
73
+ {"id": 48, "type": 6, "piece": "<0x1F>", "score": 0.0}
74
+ {"id": 49, "type": 6, "piece": "<0x20>", "score": 0.0}
75
+ {"id": 50, "type": 6, "piece": "<0x21>", "score": 0.0}
76
+ {"id": 51, "type": 6, "piece": "<0x22>", "score": 0.0}
77
+ {"id": 52, "type": 6, "piece": "<0x23>", "score": 0.0}
78
+ {"id": 53, "type": 6, "piece": "<0x24>", "score": 0.0}
79
+ {"id": 54, "type": 6, "piece": "<0x25>", "score": 0.0}
80
+ {"id": 55, "type": 6, "piece": "<0x26>", "score": 0.0}
81
+ {"id": 56, "type": 6, "piece": "<0x27>", "score": 0.0}
82
+ {"id": 57, "type": 6, "piece": "<0x28>", "score": 0.0}
83
+ {"id": 58, "type": 6, "piece": "<0x29>", "score": 0.0}
84
+ {"id": 59, "type": 6, "piece": "<0x2A>", "score": 0.0}
85
+ {"id": 60, "type": 6, "piece": "<0x2B>", "score": 0.0}
86
+ {"id": 61, "type": 6, "piece": "<0x2C>", "score": 0.0}
87
+ {"id": 62, "type": 6, "piece": "<0x2D>", "score": 0.0}
88
+ {"id": 63, "type": 6, "piece": "<0x2E>", "score": 0.0}
89
+ {"id": 64, "type": 6, "piece": "<0x2F>", "score": 0.0}
90
+ {"id": 65, "type": 6, "piece": "<0x30>", "score": 0.0}
91
+ {"id": 66, "type": 6, "piece": "<0x31>", "score": 0.0}
92
+ {"id": 67, "type": 6, "piece": "<0x32>", "score": 0.0}
93
+ {"id": 68, "type": 6, "piece": "<0x33>", "score": 0.0}
94
+ {"id": 69, "type": 6, "piece": "<0x34>", "score": 0.0}
95
+ {"id": 70, "type": 6, "piece": "<0x35>", "score": 0.0}
96
+ {"id": 71, "type": 6, "piece": "<0x36>", "score": 0.0}
97
+ {"id": 72, "type": 6, "piece": "<0x37>", "score": 0.0}
98
+ {"id": 73, "type": 6, "piece": "<0x38>", "score": 0.0}
99
+ {"id": 74, "type": 6, "piece": "<0x39>", "score": 0.0}
100
+ {"id": 75, "type": 6, "piece": "<0x3A>", "score": 0.0}
101
+ {"id": 76, "type": 6, "piece": "<0x3B>", "score": 0.0}
102
+ {"id": 77, "type": 6, "piece": "<0x3C>", "score": 0.0}
103
+ {"id": 78, "type": 6, "piece": "<0x3D>", "score": 0.0}
104
+ {"id": 79, "type": 6, "piece": "<0x3E>", "score": 0.0}
105
+ {"id": 80, "type": 6, "piece": "<0x3F>", "score": 0.0}
106
+ {"id": 81, "type": 6, "piece": "<0x40>", "score": 0.0}
107
+ {"id": 82, "type": 6, "piece": "<0x41>", "score": 0.0}
108
+ {"id": 83, "type": 6, "piece": "<0x42>", "score": 0.0}
109
+ {"id": 84, "type": 6, "piece": "<0x43>", "score": 0.0}
110
+ {"id": 85, "type": 6, "piece": "<0x44>", "score": 0.0}
111
+ {"id": 86, "type": 6, "piece": "<0x45>", "score": 0.0}
112
+ {"id": 87, "type": 6, "piece": "<0x46>", "score": 0.0}
113
+ {"id": 88, "type": 6, "piece": "<0x47>", "score": 0.0}
114
+ {"id": 89, "type": 6, "piece": "<0x48>", "score": 0.0}
115
+ {"id": 90, "type": 6, "piece": "<0x49>", "score": 0.0}
116
+ {"id": 91, "type": 6, "piece": "<0x4A>", "score": 0.0}
117
+ {"id": 92, "type": 6, "piece": "<0x4B>", "score": 0.0}
118
+ {"id": 93, "type": 6, "piece": "<0x4C>", "score": 0.0}
119
+ {"id": 94, "type": 6, "piece": "<0x4D>", "score": 0.0}
120
+ {"id": 95, "type": 6, "piece": "<0x4E>", "score": 0.0}
121
+ {"id": 96, "type": 6, "piece": "<0x4F>", "score": 0.0}
122
+ {"id": 97, "type": 6, "piece": "<0x50>", "score": 0.0}
123
+ {"id": 98, "type": 6, "piece": "<0x51>", "score": 0.0}
124
+ {"id": 99, "type": 6, "piece": "<0x52>", "score": 0.0}
125
+ {"id": 100, "type": 6, "piece": "<0x53>", "score": 0.0}
126
+ {"id": 101, "type": 6, "piece": "<0x54>", "score": 0.0}
127
+ {"id": 102, "type": 6, "piece": "<0x55>", "score": 0.0}
128
+ {"id": 103, "type": 6, "piece": "<0x56>", "score": 0.0}
129
+ {"id": 104, "type": 6, "piece": "<0x57>", "score": 0.0}
130
+ {"id": 105, "type": 6, "piece": "<0x58>", "score": 0.0}
131
+ {"id": 106, "type": 6, "piece": "<0x59>", "score": 0.0}
132
+ {"id": 107, "type": 6, "piece": "<0x5A>", "score": 0.0}
133
+ {"id": 108, "type": 6, "piece": "<0x5B>", "score": 0.0}
134
+ {"id": 109, "type": 6, "piece": "<0x5C>", "score": 0.0}
135
+ {"id": 110, "type": 6, "piece": "<0x5D>", "score": 0.0}
136
+ {"id": 111, "type": 6, "piece": "<0x5E>", "score": 0.0}
137
+ {"id": 112, "type": 6, "piece": "<0x5F>", "score": 0.0}
138
+ {"id": 113, "type": 6, "piece": "<0x60>", "score": 0.0}
139
+ {"id": 114, "type": 6, "piece": "<0x61>", "score": 0.0}
140
+ {"id": 115, "type": 6, "piece": "<0x62>", "score": 0.0}
141
+ {"id": 116, "type": 6, "piece": "<0x63>", "score": 0.0}
142
+ {"id": 117, "type": 6, "piece": "<0x64>", "score": 0.0}
143
+ {"id": 118, "type": 6, "piece": "<0x65>", "score": 0.0}
144
+ {"id": 119, "type": 6, "piece": "<0x66>", "score": 0.0}
145
+ {"id": 120, "type": 6, "piece": "<0x67>", "score": 0.0}
146
+ {"id": 121, "type": 6, "piece": "<0x68>", "score": 0.0}
147
+ {"id": 122, "type": 6, "piece": "<0x69>", "score": 0.0}
148
+ {"id": 123, "type": 6, "piece": "<0x6A>", "score": 0.0}
149
+ {"id": 124, "type": 6, "piece": "<0x6B>", "score": 0.0}
150
+ {"id": 125, "type": 6, "piece": "<0x6C>", "score": 0.0}
151
+ {"id": 126, "type": 6, "piece": "<0x6D>", "score": 0.0}
152
+ {"id": 127, "type": 6, "piece": "<0x6E>", "score": 0.0}
153
+ {"id": 128, "type": 6, "piece": "<0x6F>", "score": 0.0}
154
+ {"id": 129, "type": 6, "piece": "<0x70>", "score": 0.0}
155
+ {"id": 130, "type": 6, "piece": "<0x71>", "score": 0.0}
156
+ {"id": 131, "type": 6, "piece": "<0x72>", "score": 0.0}
157
+ {"id": 132, "type": 6, "piece": "<0x73>", "score": 0.0}
158
+ {"id": 133, "type": 6, "piece": "<0x74>", "score": 0.0}
159
+ {"id": 134, "type": 6, "piece": "<0x75>", "score": 0.0}
160
+ {"id": 135, "type": 6, "piece": "<0x76>", "score": 0.0}
161
+ {"id": 136, "type": 6, "piece": "<0x77>", "score": 0.0}
162
+ {"id": 137, "type": 6, "piece": "<0x78>", "score": 0.0}
163
+ {"id": 138, "type": 6, "piece": "<0x79>", "score": 0.0}
164
+ {"id": 139, "type": 6, "piece": "<0x7A>", "score": 0.0}
165
+ {"id": 140, "type": 6, "piece": "<0x7B>", "score": 0.0}
166
+ {"id": 141, "type": 6, "piece": "<0x7C>", "score": 0.0}
167
+ {"id": 142, "type": 6, "piece": "<0x7D>", "score": 0.0}
168
+ {"id": 143, "type": 6, "piece": "<0x7E>", "score": 0.0}
169
+ {"id": 144, "type": 6, "piece": "<0x7F>", "score": 0.0}
170
+ {"id": 145, "type": 6, "piece": "<0x80>", "score": 0.0}
171
+ {"id": 146, "type": 6, "piece": "<0x81>", "score": 0.0}
172
+ {"id": 147, "type": 6, "piece": "<0x82>", "score": 0.0}
173
+ {"id": 148, "type": 6, "piece": "<0x83>", "score": 0.0}
174
+ {"id": 149, "type": 6, "piece": "<0x84>", "score": 0.0}
175
+ {"id": 150, "type": 6, "piece": "<0x85>", "score": 0.0}
176
+ {"id": 151, "type": 6, "piece": "<0x86>", "score": 0.0}
177
+ {"id": 152, "type": 6, "piece": "<0x87>", "score": 0.0}
178
+ {"id": 153, "type": 6, "piece": "<0x88>", "score": 0.0}
179
+ {"id": 154, "type": 6, "piece": "<0x89>", "score": 0.0}
180
+ {"id": 155, "type": 6, "piece": "<0x8A>", "score": 0.0}
181
+ {"id": 156, "type": 6, "piece": "<0x8B>", "score": 0.0}
182
+ {"id": 157, "type": 6, "piece": "<0x8C>", "score": 0.0}
183
+ {"id": 158, "type": 6, "piece": "<0x8D>", "score": 0.0}
184
+ {"id": 159, "type": 6, "piece": "<0x8E>", "score": 0.0}
185
+ {"id": 160, "type": 6, "piece": "<0x8F>", "score": 0.0}
186
+ {"id": 161, "type": 6, "piece": "<0x90>", "score": 0.0}
187
+ {"id": 162, "type": 6, "piece": "<0x91>", "score": 0.0}
188
+ {"id": 163, "type": 6, "piece": "<0x92>", "score": 0.0}
189
+ {"id": 164, "type": 6, "piece": "<0x93>", "score": 0.0}
190
+ {"id": 165, "type": 6, "piece": "<0x94>", "score": 0.0}
191
+ {"id": 166, "type": 6, "piece": "<0x95>", "score": 0.0}
192
+ {"id": 167, "type": 6, "piece": "<0x96>", "score": 0.0}
193
+ {"id": 168, "type": 6, "piece": "<0x97>", "score": 0.0}
194
+ {"id": 169, "type": 6, "piece": "<0x98>", "score": 0.0}
195
+ {"id": 170, "type": 6, "piece": "<0x99>", "score": 0.0}
196
+ {"id": 171, "type": 6, "piece": "<0x9A>", "score": 0.0}
197
+ {"id": 172, "type": 6, "piece": "<0x9B>", "score": 0.0}
198
+ {"id": 173, "type": 6, "piece": "<0x9C>", "score": 0.0}
199
+ {"id": 174, "type": 6, "piece": "<0x9D>", "score": 0.0}
200
+ {"id": 175, "type": 6, "piece": "<0x9E>", "score": 0.0}
201
+ {"id": 176, "type": 6, "piece": "<0x9F>", "score": 0.0}
202
+ {"id": 177, "type": 6, "piece": "<0xA0>", "score": 0.0}
203
+ {"id": 178, "type": 6, "piece": "<0xA1>", "score": 0.0}
204
+ {"id": 179, "type": 6, "piece": "<0xA2>", "score": 0.0}
205
+ {"id": 180, "type": 6, "piece": "<0xA3>", "score": 0.0}
206
+ {"id": 181, "type": 6, "piece": "<0xA4>", "score": 0.0}
207
+ {"id": 182, "type": 6, "piece": "<0xA5>", "score": 0.0}
208
+ {"id": 183, "type": 6, "piece": "<0xA6>", "score": 0.0}
209
+ {"id": 184, "type": 6, "piece": "<0xA7>", "score": 0.0}
210
+ {"id": 185, "type": 6, "piece": "<0xA8>", "score": 0.0}
211
+ {"id": 186, "type": 6, "piece": "<0xA9>", "score": 0.0}
212
+ {"id": 187, "type": 6, "piece": "<0xAA>", "score": 0.0}
213
+ {"id": 188, "type": 6, "piece": "<0xAB>", "score": 0.0}
214
+ {"id": 189, "type": 6, "piece": "<0xAC>", "score": 0.0}
215
+ {"id": 190, "type": 6, "piece": "<0xAD>", "score": 0.0}
216
+ {"id": 191, "type": 6, "piece": "<0xAE>", "score": 0.0}
217
+ {"id": 192, "type": 6, "piece": "<0xAF>", "score": 0.0}
218
+ {"id": 193, "type": 6, "piece": "<0xB0>", "score": 0.0}
219
+ {"id": 194, "type": 6, "piece": "<0xB1>", "score": 0.0}
220
+ {"id": 195, "type": 6, "piece": "<0xB2>", "score": 0.0}
221
+ {"id": 196, "type": 6, "piece": "<0xB3>", "score": 0.0}
222
+ {"id": 197, "type": 6, "piece": "<0xB4>", "score": 0.0}
223
+ {"id": 198, "type": 6, "piece": "<0xB5>", "score": 0.0}
224
+ {"id": 199, "type": 6, "piece": "<0xB6>", "score": 0.0}
225
+ {"id": 200, "type": 6, "piece": "<0xB7>", "score": 0.0}
226
+ {"id": 201, "type": 6, "piece": "<0xB8>", "score": 0.0}
227
+ {"id": 202, "type": 6, "piece": "<0xB9>", "score": 0.0}
228
+ {"id": 203, "type": 6, "piece": "<0xBA>", "score": 0.0}
229
+ {"id": 204, "type": 6, "piece": "<0xBB>", "score": 0.0}
230
+ {"id": 205, "type": 6, "piece": "<0xBC>", "score": 0.0}
231
+ {"id": 206, "type": 6, "piece": "<0xBD>", "score": 0.0}
232
+ {"id": 207, "type": 6, "piece": "<0xBE>", "score": 0.0}
233
+ {"id": 208, "type": 6, "piece": "<0xBF>", "score": 0.0}
234
+ {"id": 209, "type": 6, "piece": "<0xC0>", "score": 0.0}
235
+ {"id": 210, "type": 6, "piece": "<0xC1>", "score": 0.0}
236
+ {"id": 211, "type": 6, "piece": "<0xC2>", "score": 0.0}
237
+ {"id": 212, "type": 6, "piece": "<0xC3>", "score": 0.0}
238
+ {"id": 213, "type": 6, "piece": "<0xC4>", "score": 0.0}
239
+ {"id": 214, "type": 6, "piece": "<0xC5>", "score": 0.0}
240
+ {"id": 215, "type": 6, "piece": "<0xC6>", "score": 0.0}
241
+ {"id": 216, "type": 6, "piece": "<0xC7>", "score": 0.0}
242
+ {"id": 217, "type": 6, "piece": "<0xC8>", "score": 0.0}
243
+ {"id": 218, "type": 6, "piece": "<0xC9>", "score": 0.0}
244
+ {"id": 219, "type": 6, "piece": "<0xCA>", "score": 0.0}
245
+ {"id": 220, "type": 6, "piece": "<0xCB>", "score": 0.0}
246
+ {"id": 221, "type": 6, "piece": "<0xCC>", "score": 0.0}
247
+ {"id": 222, "type": 6, "piece": "<0xCD>", "score": 0.0}
248
+ {"id": 223, "type": 6, "piece": "<0xCE>", "score": 0.0}
249
+ {"id": 224, "type": 6, "piece": "<0xCF>", "score": 0.0}
250
+ {"id": 225, "type": 6, "piece": "<0xD0>", "score": 0.0}
251
+ {"id": 226, "type": 6, "piece": "<0xD1>", "score": 0.0}
252
+ {"id": 227, "type": 6, "piece": "<0xD2>", "score": 0.0}
253
+ {"id": 228, "type": 6, "piece": "<0xD3>", "score": 0.0}
254
+ {"id": 229, "type": 6, "piece": "<0xD4>", "score": 0.0}
255
+ {"id": 230, "type": 6, "piece": "<0xD5>", "score": 0.0}
256
+ {"id": 231, "type": 6, "piece": "<0xD6>", "score": 0.0}
257
+ {"id": 232, "type": 6, "piece": "<0xD7>", "score": 0.0}
258
+ {"id": 233, "type": 6, "piece": "<0xD8>", "score": 0.0}
259
+ {"id": 234, "type": 6, "piece": "<0xD9>", "score": 0.0}
260
+ {"id": 235, "type": 6, "piece": "<0xDA>", "score": 0.0}
261
+ {"id": 236, "type": 6, "piece": "<0xDB>", "score": 0.0}
262
+ {"id": 237, "type": 6, "piece": "<0xDC>", "score": 0.0}
263
+ {"id": 238, "type": 6, "piece": "<0xDD>", "score": 0.0}
264
+ {"id": 239, "type": 6, "piece": "<0xDE>", "score": 0.0}
265
+ {"id": 240, "type": 6, "piece": "<0xDF>", "score": 0.0}
266
+ {"id": 241, "type": 6, "piece": "<0xE0>", "score": 0.0}
267
+ {"id": 242, "type": 6, "piece": "<0xE1>", "score": 0.0}
268
+ {"id": 243, "type": 6, "piece": "<0xE2>", "score": 0.0}
269
+ {"id": 244, "type": 6, "piece": "<0xE3>", "score": 0.0}
270
+ {"id": 245, "type": 6, "piece": "<0xE4>", "score": 0.0}
271
+ {"id": 246, "type": 6, "piece": "<0xE5>", "score": 0.0}
272
+ {"id": 247, "type": 6, "piece": "<0xE6>", "score": 0.0}
273
+ {"id": 248, "type": 6, "piece": "<0xE7>", "score": 0.0}
274
+ {"id": 249, "type": 6, "piece": "<0xE8>", "score": 0.0}
275
+ {"id": 250, "type": 6, "piece": "<0xE9>", "score": 0.0}
276
+ {"id": 251, "type": 6, "piece": "<0xEA>", "score": 0.0}
277
+ {"id": 252, "type": 6, "piece": "<0xEB>", "score": 0.0}
278
+ {"id": 253, "type": 6, "piece": "<0xEC>", "score": 0.0}
279
+ {"id": 254, "type": 6, "piece": "<0xED>", "score": 0.0}
280
+ {"id": 255, "type": 6, "piece": "<0xEE>", "score": 0.0}
281
+ {"id": 256, "type": 6, "piece": "<0xEF>", "score": 0.0}
282
+ {"id": 257, "type": 6, "piece": "<0xF0>", "score": 0.0}
283
+ {"id": 258, "type": 6, "piece": "<0xF1>", "score": 0.0}
284
+ {"id": 259, "type": 6, "piece": "<0xF2>", "score": 0.0}
285
+ {"id": 260, "type": 6, "piece": "<0xF3>", "score": 0.0}
286
+ {"id": 261, "type": 6, "piece": "<0xF4>", "score": 0.0}
287
+ {"id": 262, "type": 6, "piece": "<0xF5>", "score": 0.0}
288
+ {"id": 263, "type": 6, "piece": "<0xF6>", "score": 0.0}
289
+ {"id": 264, "type": 6, "piece": "<0xF7>", "score": 0.0}
290
+ {"id": 265, "type": 6, "piece": "<0xF8>", "score": 0.0}
291
+ {"id": 266, "type": 6, "piece": "<0xF9>", "score": 0.0}
292
+ {"id": 267, "type": 6, "piece": "<0xFA>", "score": 0.0}
293
+ {"id": 268, "type": 6, "piece": "<0xFB>", "score": 0.0}
294
+ {"id": 269, "type": 6, "piece": "<0xFC>", "score": 0.0}
295
+ {"id": 270, "type": 6, "piece": "<0xFD>", "score": 0.0}
296
+ {"id": 271, "type": 6, "piece": "<0xFE>", "score": 0.0}
297
+ {"id": 272, "type": 6, "piece": "<0xFF>", "score": 0.0}
298
+ {"id": 259, "type": 4, "piece": "<extra_id_99>", "score": 0.0}
299
+ {"id": 260, "type": 4, "piece": "<extra_id_98>", "score": 0.0}
300
+ {"id": 261, "type": 4, "piece": "<extra_id_97>", "score": 0.0}
301
+ {"id": 262, "type": 4, "piece": "<extra_id_96>", "score": 0.0}
302
+ {"id": 263, "type": 4, "piece": "<extra_id_95>", "score": 0.0}
303
+ {"id": 264, "type": 4, "piece": "<extra_id_94>", "score": 0.0}
304
+ {"id": 265, "type": 4, "piece": "<extra_id_93>", "score": 0.0}
305
+ {"id": 266, "type": 4, "piece": "<extra_id_92>", "score": 0.0}
306
+ {"id": 267, "type": 4, "piece": "<extra_id_91>", "score": 0.0}
307
+ {"id": 268, "type": 4, "piece": "<extra_id_90>", "score": 0.0}
308
+ {"id": 269, "type": 4, "piece": "<extra_id_89>", "score": 0.0}
309
+ {"id": 270, "type": 4, "piece": "<extra_id_88>", "score": 0.0}
310
+ {"id": 271, "type": 4, "piece": "<extra_id_87>", "score": 0.0}
311
+ {"id": 272, "type": 4, "piece": "<extra_id_86>", "score": 0.0}
312
+ {"id": 273, "type": 4, "piece": "<extra_id_85>", "score": 0.0}
313
+ {"id": 274, "type": 4, "piece": "<extra_id_84>", "score": 0.0}
314
+ {"id": 275, "type": 4, "piece": "<extra_id_83>", "score": 0.0}
315
+ {"id": 276, "type": 4, "piece": "<extra_id_82>", "score": 0.0}
316
+ {"id": 277, "type": 4, "piece": "<extra_id_81>", "score": 0.0}
317
+ {"id": 278, "type": 4, "piece": "<extra_id_80>", "score": 0.0}
318
+ {"id": 279, "type": 4, "piece": "<extra_id_79>", "score": 0.0}
319
+ {"id": 280, "type": 4, "piece": "<extra_id_78>", "score": 0.0}
320
+ {"id": 281, "type": 4, "piece": "<extra_id_77>", "score": 0.0}
321
+ {"id": 282, "type": 4, "piece": "<extra_id_76>", "score": 0.0}
322
+ {"id": 283, "type": 4, "piece": "<extra_id_75>", "score": 0.0}
323
+ {"id": 284, "type": 4, "piece": "<extra_id_74>", "score": 0.0}
324
+ {"id": 285, "type": 4, "piece": "<extra_id_73>", "score": 0.0}
325
+ {"id": 286, "type": 4, "piece": "<extra_id_72>", "score": 0.0}
326
+ {"id": 287, "type": 4, "piece": "<extra_id_71>", "score": 0.0}
327
+ {"id": 288, "type": 4, "piece": "<extra_id_70>", "score": 0.0}
328
+ {"id": 289, "type": 4, "piece": "<extra_id_69>", "score": 0.0}
329
+ {"id": 290, "type": 4, "piece": "<extra_id_68>", "score": 0.0}
330
+ {"id": 291, "type": 4, "piece": "<extra_id_67>", "score": 0.0}
331
+ {"id": 292, "type": 4, "piece": "<extra_id_66>", "score": 0.0}
332
+ {"id": 293, "type": 4, "piece": "<extra_id_65>", "score": 0.0}
333
+ {"id": 294, "type": 4, "piece": "<extra_id_64>", "score": 0.0}
334
+ {"id": 295, "type": 4, "piece": "<extra_id_63>", "score": 0.0}
335
+ {"id": 296, "type": 4, "piece": "<extra_id_62>", "score": 0.0}
336
+ {"id": 297, "type": 4, "piece": "<extra_id_61>", "score": 0.0}
337
+ {"id": 298, "type": 4, "piece": "<extra_id_60>", "score": 0.0}
338
+ {"id": 299, "type": 4, "piece": "<extra_id_59>", "score": 0.0}
339
+ {"id": 300, "type": 4, "piece": "<extra_id_58>", "score": 0.0}
340
+ {"id": 301, "type": 4, "piece": "<extra_id_57>", "score": 0.0}
341
+ {"id": 302, "type": 4, "piece": "<extra_id_56>", "score": 0.0}
342
+ {"id": 303, "type": 4, "piece": "<extra_id_55>", "score": 0.0}
343
+ {"id": 304, "type": 4, "piece": "<extra_id_54>", "score": 0.0}
344
+ {"id": 305, "type": 4, "piece": "<extra_id_53>", "score": 0.0}
345
+ {"id": 306, "type": 4, "piece": "<extra_id_52>", "score": 0.0}
346
+ {"id": 307, "type": 4, "piece": "<extra_id_51>", "score": 0.0}
347
+ {"id": 308, "type": 4, "piece": "<extra_id_50>", "score": 0.0}
348
+ {"id": 309, "type": 4, "piece": "<extra_id_49>", "score": 0.0}
349
+ {"id": 310, "type": 4, "piece": "<extra_id_48>", "score": 0.0}
350
+ {"id": 311, "type": 4, "piece": "<extra_id_47>", "score": 0.0}
351
+ {"id": 312, "type": 4, "piece": "<extra_id_46>", "score": 0.0}
352
+ {"id": 313, "type": 4, "piece": "<extra_id_45>", "score": 0.0}
353
+ {"id": 314, "type": 4, "piece": "<extra_id_44>", "score": 0.0}
354
+ {"id": 315, "type": 4, "piece": "<extra_id_43>", "score": 0.0}
355
+ {"id": 316, "type": 4, "piece": "<extra_id_42>", "score": 0.0}
356
+ {"id": 317, "type": 4, "piece": "<extra_id_41>", "score": 0.0}
357
+ {"id": 318, "type": 4, "piece": "<extra_id_40>", "score": 0.0}
358
+ {"id": 319, "type": 4, "piece": "<extra_id_39>", "score": 0.0}
359
+ {"id": 320, "type": 4, "piece": "<extra_id_38>", "score": 0.0}
360
+ {"id": 321, "type": 4, "piece": "<extra_id_37>", "score": 0.0}
361
+ {"id": 322, "type": 4, "piece": "<extra_id_36>", "score": 0.0}
362
+ {"id": 323, "type": 4, "piece": "<extra_id_35>", "score": 0.0}
363
+ {"id": 324, "type": 4, "piece": "<extra_id_34>", "score": 0.0}
364
+ {"id": 325, "type": 4, "piece": "<extra_id_33>", "score": 0.0}
365
+ {"id": 326, "type": 4, "piece": "<extra_id_32>", "score": 0.0}
366
+ {"id": 327, "type": 4, "piece": "<extra_id_31>", "score": 0.0}
367
+ {"id": 328, "type": 4, "piece": "<extra_id_30>", "score": 0.0}
368
+ {"id": 329, "type": 4, "piece": "<extra_id_29>", "score": 0.0}
369
+ {"id": 330, "type": 4, "piece": "<extra_id_28>", "score": 0.0}
370
+ {"id": 331, "type": 4, "piece": "<extra_id_27>", "score": 0.0}
371
+ {"id": 332, "type": 4, "piece": "<extra_id_26>", "score": 0.0}
372
+ {"id": 333, "type": 4, "piece": "<extra_id_25>", "score": 0.0}
373
+ {"id": 334, "type": 4, "piece": "<extra_id_24>", "score": 0.0}
374
+ {"id": 335, "type": 4, "piece": "<extra_id_23>", "score": 0.0}
375
+ {"id": 336, "type": 4, "piece": "<extra_id_22>", "score": 0.0}
376
+ {"id": 337, "type": 4, "piece": "<extra_id_21>", "score": 0.0}
377
+ {"id": 338, "type": 4, "piece": "<extra_id_20>", "score": 0.0}
378
+ {"id": 339, "type": 4, "piece": "<extra_id_19>", "score": 0.0}
379
+ {"id": 340, "type": 4, "piece": "<extra_id_18>", "score": 0.0}
380
+ {"id": 341, "type": 4, "piece": "<extra_id_17>", "score": 0.0}
381
+ {"id": 342, "type": 4, "piece": "<extra_id_16>", "score": 0.0}
382
+ {"id": 343, "type": 4, "piece": "<extra_id_15>", "score": 0.0}
383
+ {"id": 344, "type": 4, "piece": "<extra_id_14>", "score": 0.0}
384
+ {"id": 345, "type": 4, "piece": "<extra_id_13>", "score": 0.0}
385
+ {"id": 346, "type": 4, "piece": "<extra_id_12>", "score": 0.0}
386
+ {"id": 347, "type": 4, "piece": "<extra_id_11>", "score": 0.0}
387
+ {"id": 348, "type": 4, "piece": "<extra_id_10>", "score": 0.0}
388
+ {"id": 349, "type": 4, "piece": "<extra_id_9>", "score": 0.0}
389
+ {"id": 350, "type": 4, "piece": "<extra_id_8>", "score": 0.0}
390
+ {"id": 351, "type": 4, "piece": "<extra_id_7>", "score": 0.0}
391
+ {"id": 352, "type": 4, "piece": "<extra_id_6>", "score": 0.0}
392
+ {"id": 353, "type": 4, "piece": "<extra_id_5>", "score": 0.0}
393
+ {"id": 354, "type": 4, "piece": "<extra_id_4>", "score": 0.0}
394
+ {"id": 355, "type": 4, "piece": "<extra_id_3>", "score": 0.0}
395
+ {"id": 356, "type": 4, "piece": "<extra_id_2>", "score": 0.0}
396
+ {"id": 357, "type": 4, "piece": "<extra_id_1>", "score": 0.0}
397
+ {"id": 358, "type": 4, "piece": "<extra_id_0>", "score": 0.0}
new.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
special_tokens_map.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "eos_token": "</s>",
3
+ "pad_token": "<pad>",
4
+ "unk_token": "<unk>"
5
+ }
spiece.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ead2d948c1015cb9dd44eb30dc9c91ada08c9847f39b0226d43c3a122aa1feed
3
+ size 509431
tokenizer_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [],
3
+ "clean_up_tokenization_spaces": false,
4
+ "do_lower_case": false,
5
+ "eos_token": "</s>",
6
+ "extra_ids": 0,
7
+ "legacy": true,
8
+ "model_max_length": 1000000000000000019884624838656,
9
+ "pad_token": "<pad>",
10
+ "sp_model_kwargs": {},
11
+ "tokenizer_class": "T5Tokenizer",
12
+ "unk_token": "<unk>"
13
+ }