kkuramitsu
commited on
Commit
•
6fb8f3b
1
Parent(s):
dea97e2
Upload 5 files
Browse files- head.jsonl +397 -0
- new.jsonl +0 -0
- special_tokens_map.json +5 -0
- spiece.model +3 -0
- tokenizer_config.json +13 -0
head.jsonl
ADDED
@@ -0,0 +1,397 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{"id": 0, "type": 3, "piece": "<pad>", "score": 0.0}
|
2 |
+
{"id": 1, "type": 3, "piece": "</s>", "score": 0.0}
|
3 |
+
{"id": 2, "type": 3, "piece": "<s>", "score": 0.0}
|
4 |
+
{"id": 3, "type": 2, "piece": "<unk>", "score": 0.0}
|
5 |
+
{"id": 4, "type": 4, "piece": "<nl>", "score": 0.0}
|
6 |
+
{"id": 5, "type": 4, "piece": "<code>", "score": 0.0}
|
7 |
+
{"id": 6, "type": 4, "piece": "</code>", "score": 0.0}
|
8 |
+
{"id": 7, "type": 4, "piece": "```", "score": 0.0}
|
9 |
+
{"id": 8, "type": 4, "piece": "\"\"\"", "score": 0.0}
|
10 |
+
{"id": 9, "type": 4, "piece": "<mask>", "score": 0.0}
|
11 |
+
{"id": 10, "type": 4, "piece": "<sep>", "score": 0.0}
|
12 |
+
{"id": 11, "type": 4, "piece": "<C>", "score": 0.0}
|
13 |
+
{"id": 12, "type": 4, "piece": "▁<C>", "score": 0.0}
|
14 |
+
{"id": 13, "type": 4, "piece": "<K>", "score": 0.0}
|
15 |
+
{"id": 14, "type": 4, "piece": "<H>", "score": 0.0}
|
16 |
+
{"id": 15, "type": 4, "piece": "<X>", "score": 0.0}
|
17 |
+
{"id": 16, "type": 4, "piece": "<Y>", "score": 0.0}
|
18 |
+
{"id": 10, "type": 4, "piece": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", "score": 0.0}
|
19 |
+
{"id": 11, "type": 4, "piece": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", "score": 0.0}
|
20 |
+
{"id": 12, "type": 4, "piece": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", "score": 0.0}
|
21 |
+
{"id": 13, "type": 4, "piece": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", "score": 0.0}
|
22 |
+
{"id": 14, "type": 4, "piece": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", "score": 0.0}
|
23 |
+
{"id": 15, "type": 4, "piece": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", "score": 0.0}
|
24 |
+
{"id": 16, "type": 4, "piece": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", "score": 0.0}
|
25 |
+
{"id": 17, "type": 4, "piece": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", "score": 0.0}
|
26 |
+
{"id": 18, "type": 4, "piece": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", "score": 0.0}
|
27 |
+
{"id": 19, "type": 4, "piece": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", "score": 0.0}
|
28 |
+
{"id": 20, "type": 4, "piece": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁", "score": 0.0}
|
29 |
+
{"id": 21, "type": 4, "piece": "▁▁▁▁▁▁▁▁▁▁▁▁▁", "score": 0.0}
|
30 |
+
{"id": 22, "type": 4, "piece": "▁▁▁▁▁▁▁▁▁▁▁▁", "score": 0.0}
|
31 |
+
{"id": 23, "type": 4, "piece": "▁▁▁▁▁▁▁▁▁▁▁", "score": 0.0}
|
32 |
+
{"id": 24, "type": 4, "piece": "▁▁▁▁▁▁▁▁▁▁", "score": 0.0}
|
33 |
+
{"id": 25, "type": 4, "piece": "▁▁▁▁▁▁▁▁▁", "score": 0.0}
|
34 |
+
{"id": 26, "type": 4, "piece": "▁▁▁▁▁▁▁▁", "score": 0.0}
|
35 |
+
{"id": 27, "type": 4, "piece": "▁▁▁▁▁▁▁", "score": 0.0}
|
36 |
+
{"id": 28, "type": 4, "piece": "▁▁▁▁▁▁", "score": 0.0}
|
37 |
+
{"id": 29, "type": 4, "piece": "▁▁▁▁▁", "score": 0.0}
|
38 |
+
{"id": 30, "type": 4, "piece": "▁▁▁▁", "score": 0.0}
|
39 |
+
{"id": 31, "type": 4, "piece": "▁▁▁", "score": 0.0}
|
40 |
+
{"id": 32, "type": 4, "piece": "▁▁", "score": 0.0}
|
41 |
+
{"id": 33, "type": 4, "piece": "▁", "score": 0.0}
|
42 |
+
{"id": 17, "type": 6, "piece": "<0x00>", "score": 0.0}
|
43 |
+
{"id": 18, "type": 6, "piece": "<0x01>", "score": 0.0}
|
44 |
+
{"id": 19, "type": 6, "piece": "<0x02>", "score": 0.0}
|
45 |
+
{"id": 20, "type": 6, "piece": "<0x03>", "score": 0.0}
|
46 |
+
{"id": 21, "type": 6, "piece": "<0x04>", "score": 0.0}
|
47 |
+
{"id": 22, "type": 6, "piece": "<0x05>", "score": 0.0}
|
48 |
+
{"id": 23, "type": 6, "piece": "<0x06>", "score": 0.0}
|
49 |
+
{"id": 24, "type": 6, "piece": "<0x07>", "score": 0.0}
|
50 |
+
{"id": 25, "type": 6, "piece": "<0x08>", "score": 0.0}
|
51 |
+
{"id": 26, "type": 6, "piece": "<0x09>", "score": 0.0}
|
52 |
+
{"id": 27, "type": 6, "piece": "<0x0A>", "score": 0.0}
|
53 |
+
{"id": 28, "type": 6, "piece": "<0x0B>", "score": 0.0}
|
54 |
+
{"id": 29, "type": 6, "piece": "<0x0C>", "score": 0.0}
|
55 |
+
{"id": 30, "type": 6, "piece": "<0x0D>", "score": 0.0}
|
56 |
+
{"id": 31, "type": 6, "piece": "<0x0E>", "score": 0.0}
|
57 |
+
{"id": 32, "type": 6, "piece": "<0x0F>", "score": 0.0}
|
58 |
+
{"id": 33, "type": 6, "piece": "<0x10>", "score": 0.0}
|
59 |
+
{"id": 34, "type": 6, "piece": "<0x11>", "score": 0.0}
|
60 |
+
{"id": 35, "type": 6, "piece": "<0x12>", "score": 0.0}
|
61 |
+
{"id": 36, "type": 6, "piece": "<0x13>", "score": 0.0}
|
62 |
+
{"id": 37, "type": 6, "piece": "<0x14>", "score": 0.0}
|
63 |
+
{"id": 38, "type": 6, "piece": "<0x15>", "score": 0.0}
|
64 |
+
{"id": 39, "type": 6, "piece": "<0x16>", "score": 0.0}
|
65 |
+
{"id": 40, "type": 6, "piece": "<0x17>", "score": 0.0}
|
66 |
+
{"id": 41, "type": 6, "piece": "<0x18>", "score": 0.0}
|
67 |
+
{"id": 42, "type": 6, "piece": "<0x19>", "score": 0.0}
|
68 |
+
{"id": 43, "type": 6, "piece": "<0x1A>", "score": 0.0}
|
69 |
+
{"id": 44, "type": 6, "piece": "<0x1B>", "score": 0.0}
|
70 |
+
{"id": 45, "type": 6, "piece": "<0x1C>", "score": 0.0}
|
71 |
+
{"id": 46, "type": 6, "piece": "<0x1D>", "score": 0.0}
|
72 |
+
{"id": 47, "type": 6, "piece": "<0x1E>", "score": 0.0}
|
73 |
+
{"id": 48, "type": 6, "piece": "<0x1F>", "score": 0.0}
|
74 |
+
{"id": 49, "type": 6, "piece": "<0x20>", "score": 0.0}
|
75 |
+
{"id": 50, "type": 6, "piece": "<0x21>", "score": 0.0}
|
76 |
+
{"id": 51, "type": 6, "piece": "<0x22>", "score": 0.0}
|
77 |
+
{"id": 52, "type": 6, "piece": "<0x23>", "score": 0.0}
|
78 |
+
{"id": 53, "type": 6, "piece": "<0x24>", "score": 0.0}
|
79 |
+
{"id": 54, "type": 6, "piece": "<0x25>", "score": 0.0}
|
80 |
+
{"id": 55, "type": 6, "piece": "<0x26>", "score": 0.0}
|
81 |
+
{"id": 56, "type": 6, "piece": "<0x27>", "score": 0.0}
|
82 |
+
{"id": 57, "type": 6, "piece": "<0x28>", "score": 0.0}
|
83 |
+
{"id": 58, "type": 6, "piece": "<0x29>", "score": 0.0}
|
84 |
+
{"id": 59, "type": 6, "piece": "<0x2A>", "score": 0.0}
|
85 |
+
{"id": 60, "type": 6, "piece": "<0x2B>", "score": 0.0}
|
86 |
+
{"id": 61, "type": 6, "piece": "<0x2C>", "score": 0.0}
|
87 |
+
{"id": 62, "type": 6, "piece": "<0x2D>", "score": 0.0}
|
88 |
+
{"id": 63, "type": 6, "piece": "<0x2E>", "score": 0.0}
|
89 |
+
{"id": 64, "type": 6, "piece": "<0x2F>", "score": 0.0}
|
90 |
+
{"id": 65, "type": 6, "piece": "<0x30>", "score": 0.0}
|
91 |
+
{"id": 66, "type": 6, "piece": "<0x31>", "score": 0.0}
|
92 |
+
{"id": 67, "type": 6, "piece": "<0x32>", "score": 0.0}
|
93 |
+
{"id": 68, "type": 6, "piece": "<0x33>", "score": 0.0}
|
94 |
+
{"id": 69, "type": 6, "piece": "<0x34>", "score": 0.0}
|
95 |
+
{"id": 70, "type": 6, "piece": "<0x35>", "score": 0.0}
|
96 |
+
{"id": 71, "type": 6, "piece": "<0x36>", "score": 0.0}
|
97 |
+
{"id": 72, "type": 6, "piece": "<0x37>", "score": 0.0}
|
98 |
+
{"id": 73, "type": 6, "piece": "<0x38>", "score": 0.0}
|
99 |
+
{"id": 74, "type": 6, "piece": "<0x39>", "score": 0.0}
|
100 |
+
{"id": 75, "type": 6, "piece": "<0x3A>", "score": 0.0}
|
101 |
+
{"id": 76, "type": 6, "piece": "<0x3B>", "score": 0.0}
|
102 |
+
{"id": 77, "type": 6, "piece": "<0x3C>", "score": 0.0}
|
103 |
+
{"id": 78, "type": 6, "piece": "<0x3D>", "score": 0.0}
|
104 |
+
{"id": 79, "type": 6, "piece": "<0x3E>", "score": 0.0}
|
105 |
+
{"id": 80, "type": 6, "piece": "<0x3F>", "score": 0.0}
|
106 |
+
{"id": 81, "type": 6, "piece": "<0x40>", "score": 0.0}
|
107 |
+
{"id": 82, "type": 6, "piece": "<0x41>", "score": 0.0}
|
108 |
+
{"id": 83, "type": 6, "piece": "<0x42>", "score": 0.0}
|
109 |
+
{"id": 84, "type": 6, "piece": "<0x43>", "score": 0.0}
|
110 |
+
{"id": 85, "type": 6, "piece": "<0x44>", "score": 0.0}
|
111 |
+
{"id": 86, "type": 6, "piece": "<0x45>", "score": 0.0}
|
112 |
+
{"id": 87, "type": 6, "piece": "<0x46>", "score": 0.0}
|
113 |
+
{"id": 88, "type": 6, "piece": "<0x47>", "score": 0.0}
|
114 |
+
{"id": 89, "type": 6, "piece": "<0x48>", "score": 0.0}
|
115 |
+
{"id": 90, "type": 6, "piece": "<0x49>", "score": 0.0}
|
116 |
+
{"id": 91, "type": 6, "piece": "<0x4A>", "score": 0.0}
|
117 |
+
{"id": 92, "type": 6, "piece": "<0x4B>", "score": 0.0}
|
118 |
+
{"id": 93, "type": 6, "piece": "<0x4C>", "score": 0.0}
|
119 |
+
{"id": 94, "type": 6, "piece": "<0x4D>", "score": 0.0}
|
120 |
+
{"id": 95, "type": 6, "piece": "<0x4E>", "score": 0.0}
|
121 |
+
{"id": 96, "type": 6, "piece": "<0x4F>", "score": 0.0}
|
122 |
+
{"id": 97, "type": 6, "piece": "<0x50>", "score": 0.0}
|
123 |
+
{"id": 98, "type": 6, "piece": "<0x51>", "score": 0.0}
|
124 |
+
{"id": 99, "type": 6, "piece": "<0x52>", "score": 0.0}
|
125 |
+
{"id": 100, "type": 6, "piece": "<0x53>", "score": 0.0}
|
126 |
+
{"id": 101, "type": 6, "piece": "<0x54>", "score": 0.0}
|
127 |
+
{"id": 102, "type": 6, "piece": "<0x55>", "score": 0.0}
|
128 |
+
{"id": 103, "type": 6, "piece": "<0x56>", "score": 0.0}
|
129 |
+
{"id": 104, "type": 6, "piece": "<0x57>", "score": 0.0}
|
130 |
+
{"id": 105, "type": 6, "piece": "<0x58>", "score": 0.0}
|
131 |
+
{"id": 106, "type": 6, "piece": "<0x59>", "score": 0.0}
|
132 |
+
{"id": 107, "type": 6, "piece": "<0x5A>", "score": 0.0}
|
133 |
+
{"id": 108, "type": 6, "piece": "<0x5B>", "score": 0.0}
|
134 |
+
{"id": 109, "type": 6, "piece": "<0x5C>", "score": 0.0}
|
135 |
+
{"id": 110, "type": 6, "piece": "<0x5D>", "score": 0.0}
|
136 |
+
{"id": 111, "type": 6, "piece": "<0x5E>", "score": 0.0}
|
137 |
+
{"id": 112, "type": 6, "piece": "<0x5F>", "score": 0.0}
|
138 |
+
{"id": 113, "type": 6, "piece": "<0x60>", "score": 0.0}
|
139 |
+
{"id": 114, "type": 6, "piece": "<0x61>", "score": 0.0}
|
140 |
+
{"id": 115, "type": 6, "piece": "<0x62>", "score": 0.0}
|
141 |
+
{"id": 116, "type": 6, "piece": "<0x63>", "score": 0.0}
|
142 |
+
{"id": 117, "type": 6, "piece": "<0x64>", "score": 0.0}
|
143 |
+
{"id": 118, "type": 6, "piece": "<0x65>", "score": 0.0}
|
144 |
+
{"id": 119, "type": 6, "piece": "<0x66>", "score": 0.0}
|
145 |
+
{"id": 120, "type": 6, "piece": "<0x67>", "score": 0.0}
|
146 |
+
{"id": 121, "type": 6, "piece": "<0x68>", "score": 0.0}
|
147 |
+
{"id": 122, "type": 6, "piece": "<0x69>", "score": 0.0}
|
148 |
+
{"id": 123, "type": 6, "piece": "<0x6A>", "score": 0.0}
|
149 |
+
{"id": 124, "type": 6, "piece": "<0x6B>", "score": 0.0}
|
150 |
+
{"id": 125, "type": 6, "piece": "<0x6C>", "score": 0.0}
|
151 |
+
{"id": 126, "type": 6, "piece": "<0x6D>", "score": 0.0}
|
152 |
+
{"id": 127, "type": 6, "piece": "<0x6E>", "score": 0.0}
|
153 |
+
{"id": 128, "type": 6, "piece": "<0x6F>", "score": 0.0}
|
154 |
+
{"id": 129, "type": 6, "piece": "<0x70>", "score": 0.0}
|
155 |
+
{"id": 130, "type": 6, "piece": "<0x71>", "score": 0.0}
|
156 |
+
{"id": 131, "type": 6, "piece": "<0x72>", "score": 0.0}
|
157 |
+
{"id": 132, "type": 6, "piece": "<0x73>", "score": 0.0}
|
158 |
+
{"id": 133, "type": 6, "piece": "<0x74>", "score": 0.0}
|
159 |
+
{"id": 134, "type": 6, "piece": "<0x75>", "score": 0.0}
|
160 |
+
{"id": 135, "type": 6, "piece": "<0x76>", "score": 0.0}
|
161 |
+
{"id": 136, "type": 6, "piece": "<0x77>", "score": 0.0}
|
162 |
+
{"id": 137, "type": 6, "piece": "<0x78>", "score": 0.0}
|
163 |
+
{"id": 138, "type": 6, "piece": "<0x79>", "score": 0.0}
|
164 |
+
{"id": 139, "type": 6, "piece": "<0x7A>", "score": 0.0}
|
165 |
+
{"id": 140, "type": 6, "piece": "<0x7B>", "score": 0.0}
|
166 |
+
{"id": 141, "type": 6, "piece": "<0x7C>", "score": 0.0}
|
167 |
+
{"id": 142, "type": 6, "piece": "<0x7D>", "score": 0.0}
|
168 |
+
{"id": 143, "type": 6, "piece": "<0x7E>", "score": 0.0}
|
169 |
+
{"id": 144, "type": 6, "piece": "<0x7F>", "score": 0.0}
|
170 |
+
{"id": 145, "type": 6, "piece": "<0x80>", "score": 0.0}
|
171 |
+
{"id": 146, "type": 6, "piece": "<0x81>", "score": 0.0}
|
172 |
+
{"id": 147, "type": 6, "piece": "<0x82>", "score": 0.0}
|
173 |
+
{"id": 148, "type": 6, "piece": "<0x83>", "score": 0.0}
|
174 |
+
{"id": 149, "type": 6, "piece": "<0x84>", "score": 0.0}
|
175 |
+
{"id": 150, "type": 6, "piece": "<0x85>", "score": 0.0}
|
176 |
+
{"id": 151, "type": 6, "piece": "<0x86>", "score": 0.0}
|
177 |
+
{"id": 152, "type": 6, "piece": "<0x87>", "score": 0.0}
|
178 |
+
{"id": 153, "type": 6, "piece": "<0x88>", "score": 0.0}
|
179 |
+
{"id": 154, "type": 6, "piece": "<0x89>", "score": 0.0}
|
180 |
+
{"id": 155, "type": 6, "piece": "<0x8A>", "score": 0.0}
|
181 |
+
{"id": 156, "type": 6, "piece": "<0x8B>", "score": 0.0}
|
182 |
+
{"id": 157, "type": 6, "piece": "<0x8C>", "score": 0.0}
|
183 |
+
{"id": 158, "type": 6, "piece": "<0x8D>", "score": 0.0}
|
184 |
+
{"id": 159, "type": 6, "piece": "<0x8E>", "score": 0.0}
|
185 |
+
{"id": 160, "type": 6, "piece": "<0x8F>", "score": 0.0}
|
186 |
+
{"id": 161, "type": 6, "piece": "<0x90>", "score": 0.0}
|
187 |
+
{"id": 162, "type": 6, "piece": "<0x91>", "score": 0.0}
|
188 |
+
{"id": 163, "type": 6, "piece": "<0x92>", "score": 0.0}
|
189 |
+
{"id": 164, "type": 6, "piece": "<0x93>", "score": 0.0}
|
190 |
+
{"id": 165, "type": 6, "piece": "<0x94>", "score": 0.0}
|
191 |
+
{"id": 166, "type": 6, "piece": "<0x95>", "score": 0.0}
|
192 |
+
{"id": 167, "type": 6, "piece": "<0x96>", "score": 0.0}
|
193 |
+
{"id": 168, "type": 6, "piece": "<0x97>", "score": 0.0}
|
194 |
+
{"id": 169, "type": 6, "piece": "<0x98>", "score": 0.0}
|
195 |
+
{"id": 170, "type": 6, "piece": "<0x99>", "score": 0.0}
|
196 |
+
{"id": 171, "type": 6, "piece": "<0x9A>", "score": 0.0}
|
197 |
+
{"id": 172, "type": 6, "piece": "<0x9B>", "score": 0.0}
|
198 |
+
{"id": 173, "type": 6, "piece": "<0x9C>", "score": 0.0}
|
199 |
+
{"id": 174, "type": 6, "piece": "<0x9D>", "score": 0.0}
|
200 |
+
{"id": 175, "type": 6, "piece": "<0x9E>", "score": 0.0}
|
201 |
+
{"id": 176, "type": 6, "piece": "<0x9F>", "score": 0.0}
|
202 |
+
{"id": 177, "type": 6, "piece": "<0xA0>", "score": 0.0}
|
203 |
+
{"id": 178, "type": 6, "piece": "<0xA1>", "score": 0.0}
|
204 |
+
{"id": 179, "type": 6, "piece": "<0xA2>", "score": 0.0}
|
205 |
+
{"id": 180, "type": 6, "piece": "<0xA3>", "score": 0.0}
|
206 |
+
{"id": 181, "type": 6, "piece": "<0xA4>", "score": 0.0}
|
207 |
+
{"id": 182, "type": 6, "piece": "<0xA5>", "score": 0.0}
|
208 |
+
{"id": 183, "type": 6, "piece": "<0xA6>", "score": 0.0}
|
209 |
+
{"id": 184, "type": 6, "piece": "<0xA7>", "score": 0.0}
|
210 |
+
{"id": 185, "type": 6, "piece": "<0xA8>", "score": 0.0}
|
211 |
+
{"id": 186, "type": 6, "piece": "<0xA9>", "score": 0.0}
|
212 |
+
{"id": 187, "type": 6, "piece": "<0xAA>", "score": 0.0}
|
213 |
+
{"id": 188, "type": 6, "piece": "<0xAB>", "score": 0.0}
|
214 |
+
{"id": 189, "type": 6, "piece": "<0xAC>", "score": 0.0}
|
215 |
+
{"id": 190, "type": 6, "piece": "<0xAD>", "score": 0.0}
|
216 |
+
{"id": 191, "type": 6, "piece": "<0xAE>", "score": 0.0}
|
217 |
+
{"id": 192, "type": 6, "piece": "<0xAF>", "score": 0.0}
|
218 |
+
{"id": 193, "type": 6, "piece": "<0xB0>", "score": 0.0}
|
219 |
+
{"id": 194, "type": 6, "piece": "<0xB1>", "score": 0.0}
|
220 |
+
{"id": 195, "type": 6, "piece": "<0xB2>", "score": 0.0}
|
221 |
+
{"id": 196, "type": 6, "piece": "<0xB3>", "score": 0.0}
|
222 |
+
{"id": 197, "type": 6, "piece": "<0xB4>", "score": 0.0}
|
223 |
+
{"id": 198, "type": 6, "piece": "<0xB5>", "score": 0.0}
|
224 |
+
{"id": 199, "type": 6, "piece": "<0xB6>", "score": 0.0}
|
225 |
+
{"id": 200, "type": 6, "piece": "<0xB7>", "score": 0.0}
|
226 |
+
{"id": 201, "type": 6, "piece": "<0xB8>", "score": 0.0}
|
227 |
+
{"id": 202, "type": 6, "piece": "<0xB9>", "score": 0.0}
|
228 |
+
{"id": 203, "type": 6, "piece": "<0xBA>", "score": 0.0}
|
229 |
+
{"id": 204, "type": 6, "piece": "<0xBB>", "score": 0.0}
|
230 |
+
{"id": 205, "type": 6, "piece": "<0xBC>", "score": 0.0}
|
231 |
+
{"id": 206, "type": 6, "piece": "<0xBD>", "score": 0.0}
|
232 |
+
{"id": 207, "type": 6, "piece": "<0xBE>", "score": 0.0}
|
233 |
+
{"id": 208, "type": 6, "piece": "<0xBF>", "score": 0.0}
|
234 |
+
{"id": 209, "type": 6, "piece": "<0xC0>", "score": 0.0}
|
235 |
+
{"id": 210, "type": 6, "piece": "<0xC1>", "score": 0.0}
|
236 |
+
{"id": 211, "type": 6, "piece": "<0xC2>", "score": 0.0}
|
237 |
+
{"id": 212, "type": 6, "piece": "<0xC3>", "score": 0.0}
|
238 |
+
{"id": 213, "type": 6, "piece": "<0xC4>", "score": 0.0}
|
239 |
+
{"id": 214, "type": 6, "piece": "<0xC5>", "score": 0.0}
|
240 |
+
{"id": 215, "type": 6, "piece": "<0xC6>", "score": 0.0}
|
241 |
+
{"id": 216, "type": 6, "piece": "<0xC7>", "score": 0.0}
|
242 |
+
{"id": 217, "type": 6, "piece": "<0xC8>", "score": 0.0}
|
243 |
+
{"id": 218, "type": 6, "piece": "<0xC9>", "score": 0.0}
|
244 |
+
{"id": 219, "type": 6, "piece": "<0xCA>", "score": 0.0}
|
245 |
+
{"id": 220, "type": 6, "piece": "<0xCB>", "score": 0.0}
|
246 |
+
{"id": 221, "type": 6, "piece": "<0xCC>", "score": 0.0}
|
247 |
+
{"id": 222, "type": 6, "piece": "<0xCD>", "score": 0.0}
|
248 |
+
{"id": 223, "type": 6, "piece": "<0xCE>", "score": 0.0}
|
249 |
+
{"id": 224, "type": 6, "piece": "<0xCF>", "score": 0.0}
|
250 |
+
{"id": 225, "type": 6, "piece": "<0xD0>", "score": 0.0}
|
251 |
+
{"id": 226, "type": 6, "piece": "<0xD1>", "score": 0.0}
|
252 |
+
{"id": 227, "type": 6, "piece": "<0xD2>", "score": 0.0}
|
253 |
+
{"id": 228, "type": 6, "piece": "<0xD3>", "score": 0.0}
|
254 |
+
{"id": 229, "type": 6, "piece": "<0xD4>", "score": 0.0}
|
255 |
+
{"id": 230, "type": 6, "piece": "<0xD5>", "score": 0.0}
|
256 |
+
{"id": 231, "type": 6, "piece": "<0xD6>", "score": 0.0}
|
257 |
+
{"id": 232, "type": 6, "piece": "<0xD7>", "score": 0.0}
|
258 |
+
{"id": 233, "type": 6, "piece": "<0xD8>", "score": 0.0}
|
259 |
+
{"id": 234, "type": 6, "piece": "<0xD9>", "score": 0.0}
|
260 |
+
{"id": 235, "type": 6, "piece": "<0xDA>", "score": 0.0}
|
261 |
+
{"id": 236, "type": 6, "piece": "<0xDB>", "score": 0.0}
|
262 |
+
{"id": 237, "type": 6, "piece": "<0xDC>", "score": 0.0}
|
263 |
+
{"id": 238, "type": 6, "piece": "<0xDD>", "score": 0.0}
|
264 |
+
{"id": 239, "type": 6, "piece": "<0xDE>", "score": 0.0}
|
265 |
+
{"id": 240, "type": 6, "piece": "<0xDF>", "score": 0.0}
|
266 |
+
{"id": 241, "type": 6, "piece": "<0xE0>", "score": 0.0}
|
267 |
+
{"id": 242, "type": 6, "piece": "<0xE1>", "score": 0.0}
|
268 |
+
{"id": 243, "type": 6, "piece": "<0xE2>", "score": 0.0}
|
269 |
+
{"id": 244, "type": 6, "piece": "<0xE3>", "score": 0.0}
|
270 |
+
{"id": 245, "type": 6, "piece": "<0xE4>", "score": 0.0}
|
271 |
+
{"id": 246, "type": 6, "piece": "<0xE5>", "score": 0.0}
|
272 |
+
{"id": 247, "type": 6, "piece": "<0xE6>", "score": 0.0}
|
273 |
+
{"id": 248, "type": 6, "piece": "<0xE7>", "score": 0.0}
|
274 |
+
{"id": 249, "type": 6, "piece": "<0xE8>", "score": 0.0}
|
275 |
+
{"id": 250, "type": 6, "piece": "<0xE9>", "score": 0.0}
|
276 |
+
{"id": 251, "type": 6, "piece": "<0xEA>", "score": 0.0}
|
277 |
+
{"id": 252, "type": 6, "piece": "<0xEB>", "score": 0.0}
|
278 |
+
{"id": 253, "type": 6, "piece": "<0xEC>", "score": 0.0}
|
279 |
+
{"id": 254, "type": 6, "piece": "<0xED>", "score": 0.0}
|
280 |
+
{"id": 255, "type": 6, "piece": "<0xEE>", "score": 0.0}
|
281 |
+
{"id": 256, "type": 6, "piece": "<0xEF>", "score": 0.0}
|
282 |
+
{"id": 257, "type": 6, "piece": "<0xF0>", "score": 0.0}
|
283 |
+
{"id": 258, "type": 6, "piece": "<0xF1>", "score": 0.0}
|
284 |
+
{"id": 259, "type": 6, "piece": "<0xF2>", "score": 0.0}
|
285 |
+
{"id": 260, "type": 6, "piece": "<0xF3>", "score": 0.0}
|
286 |
+
{"id": 261, "type": 6, "piece": "<0xF4>", "score": 0.0}
|
287 |
+
{"id": 262, "type": 6, "piece": "<0xF5>", "score": 0.0}
|
288 |
+
{"id": 263, "type": 6, "piece": "<0xF6>", "score": 0.0}
|
289 |
+
{"id": 264, "type": 6, "piece": "<0xF7>", "score": 0.0}
|
290 |
+
{"id": 265, "type": 6, "piece": "<0xF8>", "score": 0.0}
|
291 |
+
{"id": 266, "type": 6, "piece": "<0xF9>", "score": 0.0}
|
292 |
+
{"id": 267, "type": 6, "piece": "<0xFA>", "score": 0.0}
|
293 |
+
{"id": 268, "type": 6, "piece": "<0xFB>", "score": 0.0}
|
294 |
+
{"id": 269, "type": 6, "piece": "<0xFC>", "score": 0.0}
|
295 |
+
{"id": 270, "type": 6, "piece": "<0xFD>", "score": 0.0}
|
296 |
+
{"id": 271, "type": 6, "piece": "<0xFE>", "score": 0.0}
|
297 |
+
{"id": 272, "type": 6, "piece": "<0xFF>", "score": 0.0}
|
298 |
+
{"id": 259, "type": 4, "piece": "<extra_id_99>", "score": 0.0}
|
299 |
+
{"id": 260, "type": 4, "piece": "<extra_id_98>", "score": 0.0}
|
300 |
+
{"id": 261, "type": 4, "piece": "<extra_id_97>", "score": 0.0}
|
301 |
+
{"id": 262, "type": 4, "piece": "<extra_id_96>", "score": 0.0}
|
302 |
+
{"id": 263, "type": 4, "piece": "<extra_id_95>", "score": 0.0}
|
303 |
+
{"id": 264, "type": 4, "piece": "<extra_id_94>", "score": 0.0}
|
304 |
+
{"id": 265, "type": 4, "piece": "<extra_id_93>", "score": 0.0}
|
305 |
+
{"id": 266, "type": 4, "piece": "<extra_id_92>", "score": 0.0}
|
306 |
+
{"id": 267, "type": 4, "piece": "<extra_id_91>", "score": 0.0}
|
307 |
+
{"id": 268, "type": 4, "piece": "<extra_id_90>", "score": 0.0}
|
308 |
+
{"id": 269, "type": 4, "piece": "<extra_id_89>", "score": 0.0}
|
309 |
+
{"id": 270, "type": 4, "piece": "<extra_id_88>", "score": 0.0}
|
310 |
+
{"id": 271, "type": 4, "piece": "<extra_id_87>", "score": 0.0}
|
311 |
+
{"id": 272, "type": 4, "piece": "<extra_id_86>", "score": 0.0}
|
312 |
+
{"id": 273, "type": 4, "piece": "<extra_id_85>", "score": 0.0}
|
313 |
+
{"id": 274, "type": 4, "piece": "<extra_id_84>", "score": 0.0}
|
314 |
+
{"id": 275, "type": 4, "piece": "<extra_id_83>", "score": 0.0}
|
315 |
+
{"id": 276, "type": 4, "piece": "<extra_id_82>", "score": 0.0}
|
316 |
+
{"id": 277, "type": 4, "piece": "<extra_id_81>", "score": 0.0}
|
317 |
+
{"id": 278, "type": 4, "piece": "<extra_id_80>", "score": 0.0}
|
318 |
+
{"id": 279, "type": 4, "piece": "<extra_id_79>", "score": 0.0}
|
319 |
+
{"id": 280, "type": 4, "piece": "<extra_id_78>", "score": 0.0}
|
320 |
+
{"id": 281, "type": 4, "piece": "<extra_id_77>", "score": 0.0}
|
321 |
+
{"id": 282, "type": 4, "piece": "<extra_id_76>", "score": 0.0}
|
322 |
+
{"id": 283, "type": 4, "piece": "<extra_id_75>", "score": 0.0}
|
323 |
+
{"id": 284, "type": 4, "piece": "<extra_id_74>", "score": 0.0}
|
324 |
+
{"id": 285, "type": 4, "piece": "<extra_id_73>", "score": 0.0}
|
325 |
+
{"id": 286, "type": 4, "piece": "<extra_id_72>", "score": 0.0}
|
326 |
+
{"id": 287, "type": 4, "piece": "<extra_id_71>", "score": 0.0}
|
327 |
+
{"id": 288, "type": 4, "piece": "<extra_id_70>", "score": 0.0}
|
328 |
+
{"id": 289, "type": 4, "piece": "<extra_id_69>", "score": 0.0}
|
329 |
+
{"id": 290, "type": 4, "piece": "<extra_id_68>", "score": 0.0}
|
330 |
+
{"id": 291, "type": 4, "piece": "<extra_id_67>", "score": 0.0}
|
331 |
+
{"id": 292, "type": 4, "piece": "<extra_id_66>", "score": 0.0}
|
332 |
+
{"id": 293, "type": 4, "piece": "<extra_id_65>", "score": 0.0}
|
333 |
+
{"id": 294, "type": 4, "piece": "<extra_id_64>", "score": 0.0}
|
334 |
+
{"id": 295, "type": 4, "piece": "<extra_id_63>", "score": 0.0}
|
335 |
+
{"id": 296, "type": 4, "piece": "<extra_id_62>", "score": 0.0}
|
336 |
+
{"id": 297, "type": 4, "piece": "<extra_id_61>", "score": 0.0}
|
337 |
+
{"id": 298, "type": 4, "piece": "<extra_id_60>", "score": 0.0}
|
338 |
+
{"id": 299, "type": 4, "piece": "<extra_id_59>", "score": 0.0}
|
339 |
+
{"id": 300, "type": 4, "piece": "<extra_id_58>", "score": 0.0}
|
340 |
+
{"id": 301, "type": 4, "piece": "<extra_id_57>", "score": 0.0}
|
341 |
+
{"id": 302, "type": 4, "piece": "<extra_id_56>", "score": 0.0}
|
342 |
+
{"id": 303, "type": 4, "piece": "<extra_id_55>", "score": 0.0}
|
343 |
+
{"id": 304, "type": 4, "piece": "<extra_id_54>", "score": 0.0}
|
344 |
+
{"id": 305, "type": 4, "piece": "<extra_id_53>", "score": 0.0}
|
345 |
+
{"id": 306, "type": 4, "piece": "<extra_id_52>", "score": 0.0}
|
346 |
+
{"id": 307, "type": 4, "piece": "<extra_id_51>", "score": 0.0}
|
347 |
+
{"id": 308, "type": 4, "piece": "<extra_id_50>", "score": 0.0}
|
348 |
+
{"id": 309, "type": 4, "piece": "<extra_id_49>", "score": 0.0}
|
349 |
+
{"id": 310, "type": 4, "piece": "<extra_id_48>", "score": 0.0}
|
350 |
+
{"id": 311, "type": 4, "piece": "<extra_id_47>", "score": 0.0}
|
351 |
+
{"id": 312, "type": 4, "piece": "<extra_id_46>", "score": 0.0}
|
352 |
+
{"id": 313, "type": 4, "piece": "<extra_id_45>", "score": 0.0}
|
353 |
+
{"id": 314, "type": 4, "piece": "<extra_id_44>", "score": 0.0}
|
354 |
+
{"id": 315, "type": 4, "piece": "<extra_id_43>", "score": 0.0}
|
355 |
+
{"id": 316, "type": 4, "piece": "<extra_id_42>", "score": 0.0}
|
356 |
+
{"id": 317, "type": 4, "piece": "<extra_id_41>", "score": 0.0}
|
357 |
+
{"id": 318, "type": 4, "piece": "<extra_id_40>", "score": 0.0}
|
358 |
+
{"id": 319, "type": 4, "piece": "<extra_id_39>", "score": 0.0}
|
359 |
+
{"id": 320, "type": 4, "piece": "<extra_id_38>", "score": 0.0}
|
360 |
+
{"id": 321, "type": 4, "piece": "<extra_id_37>", "score": 0.0}
|
361 |
+
{"id": 322, "type": 4, "piece": "<extra_id_36>", "score": 0.0}
|
362 |
+
{"id": 323, "type": 4, "piece": "<extra_id_35>", "score": 0.0}
|
363 |
+
{"id": 324, "type": 4, "piece": "<extra_id_34>", "score": 0.0}
|
364 |
+
{"id": 325, "type": 4, "piece": "<extra_id_33>", "score": 0.0}
|
365 |
+
{"id": 326, "type": 4, "piece": "<extra_id_32>", "score": 0.0}
|
366 |
+
{"id": 327, "type": 4, "piece": "<extra_id_31>", "score": 0.0}
|
367 |
+
{"id": 328, "type": 4, "piece": "<extra_id_30>", "score": 0.0}
|
368 |
+
{"id": 329, "type": 4, "piece": "<extra_id_29>", "score": 0.0}
|
369 |
+
{"id": 330, "type": 4, "piece": "<extra_id_28>", "score": 0.0}
|
370 |
+
{"id": 331, "type": 4, "piece": "<extra_id_27>", "score": 0.0}
|
371 |
+
{"id": 332, "type": 4, "piece": "<extra_id_26>", "score": 0.0}
|
372 |
+
{"id": 333, "type": 4, "piece": "<extra_id_25>", "score": 0.0}
|
373 |
+
{"id": 334, "type": 4, "piece": "<extra_id_24>", "score": 0.0}
|
374 |
+
{"id": 335, "type": 4, "piece": "<extra_id_23>", "score": 0.0}
|
375 |
+
{"id": 336, "type": 4, "piece": "<extra_id_22>", "score": 0.0}
|
376 |
+
{"id": 337, "type": 4, "piece": "<extra_id_21>", "score": 0.0}
|
377 |
+
{"id": 338, "type": 4, "piece": "<extra_id_20>", "score": 0.0}
|
378 |
+
{"id": 339, "type": 4, "piece": "<extra_id_19>", "score": 0.0}
|
379 |
+
{"id": 340, "type": 4, "piece": "<extra_id_18>", "score": 0.0}
|
380 |
+
{"id": 341, "type": 4, "piece": "<extra_id_17>", "score": 0.0}
|
381 |
+
{"id": 342, "type": 4, "piece": "<extra_id_16>", "score": 0.0}
|
382 |
+
{"id": 343, "type": 4, "piece": "<extra_id_15>", "score": 0.0}
|
383 |
+
{"id": 344, "type": 4, "piece": "<extra_id_14>", "score": 0.0}
|
384 |
+
{"id": 345, "type": 4, "piece": "<extra_id_13>", "score": 0.0}
|
385 |
+
{"id": 346, "type": 4, "piece": "<extra_id_12>", "score": 0.0}
|
386 |
+
{"id": 347, "type": 4, "piece": "<extra_id_11>", "score": 0.0}
|
387 |
+
{"id": 348, "type": 4, "piece": "<extra_id_10>", "score": 0.0}
|
388 |
+
{"id": 349, "type": 4, "piece": "<extra_id_9>", "score": 0.0}
|
389 |
+
{"id": 350, "type": 4, "piece": "<extra_id_8>", "score": 0.0}
|
390 |
+
{"id": 351, "type": 4, "piece": "<extra_id_7>", "score": 0.0}
|
391 |
+
{"id": 352, "type": 4, "piece": "<extra_id_6>", "score": 0.0}
|
392 |
+
{"id": 353, "type": 4, "piece": "<extra_id_5>", "score": 0.0}
|
393 |
+
{"id": 354, "type": 4, "piece": "<extra_id_4>", "score": 0.0}
|
394 |
+
{"id": 355, "type": 4, "piece": "<extra_id_3>", "score": 0.0}
|
395 |
+
{"id": 356, "type": 4, "piece": "<extra_id_2>", "score": 0.0}
|
396 |
+
{"id": 357, "type": 4, "piece": "<extra_id_1>", "score": 0.0}
|
397 |
+
{"id": 358, "type": 4, "piece": "<extra_id_0>", "score": 0.0}
|
new.jsonl
ADDED
The diff for this file is too large to render.
See raw diff
|
|
special_tokens_map.json
ADDED
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"eos_token": "</s>",
|
3 |
+
"pad_token": "<pad>",
|
4 |
+
"unk_token": "<unk>"
|
5 |
+
}
|
spiece.model
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:ead2d948c1015cb9dd44eb30dc9c91ada08c9847f39b0226d43c3a122aa1feed
|
3 |
+
size 509431
|
tokenizer_config.json
ADDED
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"additional_special_tokens": [],
|
3 |
+
"clean_up_tokenization_spaces": false,
|
4 |
+
"do_lower_case": false,
|
5 |
+
"eos_token": "</s>",
|
6 |
+
"extra_ids": 0,
|
7 |
+
"legacy": true,
|
8 |
+
"model_max_length": 1000000000000000019884624838656,
|
9 |
+
"pad_token": "<pad>",
|
10 |
+
"sp_model_kwargs": {},
|
11 |
+
"tokenizer_class": "T5Tokenizer",
|
12 |
+
"unk_token": "<unk>"
|
13 |
+
}
|