Hugging Face's logo

PleIAs
/
Pleias-Pico

Safetensors
llama
Model card Files Community
Pclanglais commited on
Commit
3fc3cd7
•
1 Parent(s): e2c75d6

Upload folder using huggingface_hub

Browse files
Files changed (4) hide show
  1. model.safetensors +1 -1
  2. special_tokens_map.json +23 -7
  3. tokenizer.json +156 -44
  4. tokenizer_config.json +161 -17
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:9ccd5f97a42a2f14577ef345edbfca238c0064f9206d1d2b232cdc24b834d6b1
3
  size 706875632
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eb0b6175b73c76a3fa9c271b59fe68c5f79a36116b06a41b91d360deaa28b186
3
  size 706875632
special_tokens_map.json CHANGED
@@ -1,14 +1,30 @@
1
  {
2
  "additional_special_tokens": [
3
- "<|query_start|>",
4
- "<|query_end|>",
5
- "<|source_start|>",
6
- "<|source_end|>",
7
- "<|source_id_start|>",
8
- "<|source_id_end|>",
9
  "<|source_analysis_start|>",
10
  "<|source_analysis_end|>",
 
 
 
 
11
  "<|answer_start|>",
12
- "<|answer_end|>"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
13
  ]
14
  }
 
1
  {
2
  "additional_special_tokens": [
3
+ "<|tool_call_start|>",
4
+ "<|tool_call_end|>",
5
+ "<|tool_list_start|>",
6
+ "<|tool_list_end|>",
 
 
7
  "<|source_analysis_start|>",
8
  "<|source_analysis_end|>",
9
+ "<|source_start|>",
10
+ "<|source_end|>",
11
+ "<|im_start|>",
12
+ "<|im_end|>",
13
  "<|answer_start|>",
14
+ "<|answer_end|>",
15
+ "<|text_start|>",
16
+ "<|text_end|>",
17
+ "<|translation_start|>",
18
+ "<|translation_end|>",
19
+ "<|back_translation_start|>",
20
+ "<|back_translation_end|>",
21
+ "<|ocr_correction_start|>",
22
+ "<|ocr_correction_end|>",
23
+ "<|json_scheme_start|>",
24
+ "<|json_scheme_end|>",
25
+ "<|source_interpretation_start|>",
26
+ "<|source_interpretation_end|>",
27
+ "<|query_start|>",
28
+ "<|query_end|>"
29
  ]
30
  }
tokenizer.json CHANGED
@@ -39,9 +39,153 @@
39
  "normalized": false,
40
  "special": true
41
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
42
  {
43
  "id": 65526,
44
- "content": "<|query_start|>",
45
  "single_word": false,
46
  "lstrip": false,
47
  "rstrip": false,
@@ -50,7 +194,7 @@
50
  },
51
  {
52
  "id": 65527,
53
- "content": "<|query_end|>",
54
  "single_word": false,
55
  "lstrip": false,
56
  "rstrip": false,
@@ -59,7 +203,7 @@
59
  },
60
  {
61
  "id": 65528,
62
- "content": "<|source_start|>",
63
  "single_word": false,
64
  "lstrip": false,
65
  "rstrip": false,
@@ -68,7 +212,7 @@
68
  },
69
  {
70
  "id": 65529,
71
- "content": "<|source_end|>",
72
  "single_word": false,
73
  "lstrip": false,
74
  "rstrip": false,
@@ -77,7 +221,7 @@
77
  },
78
  {
79
  "id": 65530,
80
- "content": "<|source_id_start|>",
81
  "single_word": false,
82
  "lstrip": false,
83
  "rstrip": false,
@@ -86,7 +230,7 @@
86
  },
87
  {
88
  "id": 65531,
89
- "content": "<|source_id_end|>",
90
  "single_word": false,
91
  "lstrip": false,
92
  "rstrip": false,
@@ -95,7 +239,7 @@
95
  },
96
  {
97
  "id": 65532,
98
- "content": "<|source_analysis_start|>",
99
  "single_word": false,
100
  "lstrip": false,
101
  "rstrip": false,
@@ -104,7 +248,7 @@
104
  },
105
  {
106
  "id": 65533,
107
- "content": "<|source_analysis_end|>",
108
  "single_word": false,
109
  "lstrip": false,
110
  "rstrip": false,
@@ -113,7 +257,7 @@
113
  },
114
  {
115
  "id": 65534,
116
- "content": "<|answer_start|>",
117
  "single_word": false,
118
  "lstrip": false,
119
  "rstrip": false,
@@ -122,7 +266,7 @@
122
  },
123
  {
124
  "id": 65535,
125
- "content": "<|answer_end|>",
126
  "single_word": false,
127
  "lstrip": false,
128
  "rstrip": false,
@@ -65767,23 +65911,7 @@
65767
  "reste": 65506,
65768
  "ĠMonat": 65507,
65769
  "å·¥ä½ľ": 65508,
65770
- "Ġdémocratique": 65509,
65771
- "Ġsejam": 65510,
65772
- "ÑĥбеÑĢ": 65511,
65773
- "Ġaccessoires": 65512,
65774
- "Ġ709": 65513,
65775
- "Ġménages": 65514,
65776
- "Ġprodukty": 65515,
65777
- "Ġsuum": 65516,
65778
- "Ġinfluenc": 65517,
65779
- "Ġcatching": 65518,
65780
- "ĠÏĮÏĦαν": 65519,
65781
- "737": 65520,
65782
- "ĠоÑĤде": 65521,
65783
- "Ġë°ı": 65522,
65784
- "symptoms": 65523,
65785
- "apac": 65524,
65786
- "Ġdépou": 65525
65787
  },
65788
  "merges": [
65789
  "Ġ Ġ",
@@ -131048,23 +131176,7 @@
131048
  "res te",
131049
  "ĠMon at",
131050
  "å·¥ ä½ľ",
131051
- "Ġdémocra tique",
131052
- "Ġse jam",
131053
- "Ñĥб еÑĢ",
131054
- "Ġaccess oires",
131055
- "Ġ7 09",
131056
- "Ġmén ages",
131057
- "Ġproduk ty",
131058
- "Ġsu um",
131059
- "Ġinflu enc",
131060
- "Ġcatch ing",
131061
- "ĠÏĮ ÏĦαν",
131062
- "7 37",
131063
- "ĠоÑĤ де",
131064
- "Ġë° ı",
131065
- "sym ptoms",
131066
- "ap ac",
131067
- "Ġdép ou"
131068
  ]
131069
  }
131070
  }
 
39
  "normalized": false,
40
  "special": true
41
  },
42
+ {
43
+ "id": 65510,
44
+ "content": "<|tool_call_start|>",
45
+ "single_word": false,
46
+ "lstrip": false,
47
+ "rstrip": false,
48
+ "normalized": false,
49
+ "special": true
50
+ },
51
+ {
52
+ "id": 65511,
53
+ "content": "<|tool_call_end|>",
54
+ "single_word": false,
55
+ "lstrip": false,
56
+ "rstrip": false,
57
+ "normalized": false,
58
+ "special": true
59
+ },
60
+ {
61
+ "id": 65512,
62
+ "content": "<|tool_list_start|>",
63
+ "single_word": false,
64
+ "lstrip": false,
65
+ "rstrip": false,
66
+ "normalized": false,
67
+ "special": true
68
+ },
69
+ {
70
+ "id": 65513,
71
+ "content": "<|tool_list_end|>",
72
+ "single_word": false,
73
+ "lstrip": false,
74
+ "rstrip": false,
75
+ "normalized": false,
76
+ "special": true
77
+ },
78
+ {
79
+ "id": 65514,
80
+ "content": "<|source_analysis_start|>",
81
+ "single_word": false,
82
+ "lstrip": false,
83
+ "rstrip": false,
84
+ "normalized": false,
85
+ "special": true
86
+ },
87
+ {
88
+ "id": 65515,
89
+ "content": "<|source_analysis_end|>",
90
+ "single_word": false,
91
+ "lstrip": false,
92
+ "rstrip": false,
93
+ "normalized": false,
94
+ "special": true
95
+ },
96
+ {
97
+ "id": 65516,
98
+ "content": "<|source_start|>",
99
+ "single_word": false,
100
+ "lstrip": false,
101
+ "rstrip": false,
102
+ "normalized": false,
103
+ "special": true
104
+ },
105
+ {
106
+ "id": 65517,
107
+ "content": "<|source_end|>",
108
+ "single_word": false,
109
+ "lstrip": false,
110
+ "rstrip": false,
111
+ "normalized": false,
112
+ "special": true
113
+ },
114
+ {
115
+ "id": 65518,
116
+ "content": "<|im_start|>",
117
+ "single_word": false,
118
+ "lstrip": false,
119
+ "rstrip": false,
120
+ "normalized": false,
121
+ "special": true
122
+ },
123
+ {
124
+ "id": 65519,
125
+ "content": "<|im_end|>",
126
+ "single_word": false,
127
+ "lstrip": false,
128
+ "rstrip": false,
129
+ "normalized": false,
130
+ "special": true
131
+ },
132
+ {
133
+ "id": 65520,
134
+ "content": "<|answer_start|>",
135
+ "single_word": false,
136
+ "lstrip": false,
137
+ "rstrip": false,
138
+ "normalized": false,
139
+ "special": true
140
+ },
141
+ {
142
+ "id": 65521,
143
+ "content": "<|answer_end|>",
144
+ "single_word": false,
145
+ "lstrip": false,
146
+ "rstrip": false,
147
+ "normalized": false,
148
+ "special": true
149
+ },
150
+ {
151
+ "id": 65522,
152
+ "content": "<|text_start|>",
153
+ "single_word": false,
154
+ "lstrip": false,
155
+ "rstrip": false,
156
+ "normalized": false,
157
+ "special": true
158
+ },
159
+ {
160
+ "id": 65523,
161
+ "content": "<|text_end|>",
162
+ "single_word": false,
163
+ "lstrip": false,
164
+ "rstrip": false,
165
+ "normalized": false,
166
+ "special": true
167
+ },
168
+ {
169
+ "id": 65524,
170
+ "content": "<|translation_start|>",
171
+ "single_word": false,
172
+ "lstrip": false,
173
+ "rstrip": false,
174
+ "normalized": false,
175
+ "special": true
176
+ },
177
+ {
178
+ "id": 65525,
179
+ "content": "<|translation_end|>",
180
+ "single_word": false,
181
+ "lstrip": false,
182
+ "rstrip": false,
183
+ "normalized": false,
184
+ "special": true
185
+ },
186
  {
187
  "id": 65526,
188
+ "content": "<|back_translation_start|>",
189
  "single_word": false,
190
  "lstrip": false,
191
  "rstrip": false,
 
194
  },
195
  {
196
  "id": 65527,
197
+ "content": "<|back_translation_end|>",
198
  "single_word": false,
199
  "lstrip": false,
200
  "rstrip": false,
 
203
  },
204
  {
205
  "id": 65528,
206
+ "content": "<|ocr_correction_start|>",
207
  "single_word": false,
208
  "lstrip": false,
209
  "rstrip": false,
 
212
  },
213
  {
214
  "id": 65529,
215
+ "content": "<|ocr_correction_end|>",
216
  "single_word": false,
217
  "lstrip": false,
218
  "rstrip": false,
 
221
  },
222
  {
223
  "id": 65530,
224
+ "content": "<|json_scheme_start|>",
225
  "single_word": false,
226
  "lstrip": false,
227
  "rstrip": false,
 
230
  },
231
  {
232
  "id": 65531,
233
+ "content": "<|json_scheme_end|>",
234
  "single_word": false,
235
  "lstrip": false,
236
  "rstrip": false,
 
239
  },
240
  {
241
  "id": 65532,
242
+ "content": "<|source_interpretation_start|>",
243
  "single_word": false,
244
  "lstrip": false,
245
  "rstrip": false,
 
248
  },
249
  {
250
  "id": 65533,
251
+ "content": "<|source_interpretation_end|>",
252
  "single_word": false,
253
  "lstrip": false,
254
  "rstrip": false,
 
257
  },
258
  {
259
  "id": 65534,
260
+ "content": "<|query_start|>",
261
  "single_word": false,
262
  "lstrip": false,
263
  "rstrip": false,
 
266
  },
267
  {
268
  "id": 65535,
269
+ "content": "<|query_end|>",
270
  "single_word": false,
271
  "lstrip": false,
272
  "rstrip": false,
 
65911
  "reste": 65506,
65912
  "ĠMonat": 65507,
65913
  "å·¥ä½ľ": 65508,
65914
+ "Ġdémocratique": 65509
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
65915
  },
65916
  "merges": [
65917
  "Ġ Ġ",
 
131176
  "res te",
131177
  "ĠMon at",
131178
  "å·¥ ä½ľ",
131179
+ "Ġdémocra tique"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
131180
  ]
131181
  }
131182
  }
tokenizer_config.json CHANGED
@@ -32,8 +32,136 @@
32
  "single_word": false,
33
  "special": true
34
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
35
  "65526": {
36
- "content": "<|query_start|>",
37
  "lstrip": false,
38
  "normalized": false,
39
  "rstrip": false,
@@ -41,7 +169,7 @@
41
  "special": true
42
  },
43
  "65527": {
44
- "content": "<|query_end|>",
45
  "lstrip": false,
46
  "normalized": false,
47
  "rstrip": false,
@@ -49,7 +177,7 @@
49
  "special": true
50
  },
51
  "65528": {
52
- "content": "<|source_start|>",
53
  "lstrip": false,
54
  "normalized": false,
55
  "rstrip": false,
@@ -57,7 +185,7 @@
57
  "special": true
58
  },
59
  "65529": {
60
- "content": "<|source_end|>",
61
  "lstrip": false,
62
  "normalized": false,
63
  "rstrip": false,
@@ -65,7 +193,7 @@
65
  "special": true
66
  },
67
  "65530": {
68
- "content": "<|source_id_start|>",
69
  "lstrip": false,
70
  "normalized": false,
71
  "rstrip": false,
@@ -73,7 +201,7 @@
73
  "special": true
74
  },
75
  "65531": {
76
- "content": "<|source_id_end|>",
77
  "lstrip": false,
78
  "normalized": false,
79
  "rstrip": false,
@@ -81,7 +209,7 @@
81
  "special": true
82
  },
83
  "65532": {
84
- "content": "<|source_analysis_start|>",
85
  "lstrip": false,
86
  "normalized": false,
87
  "rstrip": false,
@@ -89,7 +217,7 @@
89
  "special": true
90
  },
91
  "65533": {
92
- "content": "<|source_analysis_end|>",
93
  "lstrip": false,
94
  "normalized": false,
95
  "rstrip": false,
@@ -97,7 +225,7 @@
97
  "special": true
98
  },
99
  "65534": {
100
- "content": "<|answer_start|>",
101
  "lstrip": false,
102
  "normalized": false,
103
  "rstrip": false,
@@ -105,7 +233,7 @@
105
  "special": true
106
  },
107
  "65535": {
108
- "content": "<|answer_end|>",
109
  "lstrip": false,
110
  "normalized": false,
111
  "rstrip": false,
@@ -114,16 +242,32 @@
114
  }
115
  },
116
  "additional_special_tokens": [
117
- "<|query_start|>",
118
- "<|query_end|>",
119
- "<|source_start|>",
120
- "<|source_end|>",
121
- "<|source_id_start|>",
122
- "<|source_id_end|>",
123
  "<|source_analysis_start|>",
124
  "<|source_analysis_end|>",
 
 
 
 
125
  "<|answer_start|>",
126
- "<|answer_end|>"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
127
  ],
128
  "clean_up_tokenization_spaces": true,
129
  "model_max_length": 1000000000000000019884624838656,
 
32
  "single_word": false,
33
  "special": true
34
  },
35
+ "65510": {
36
+ "content": "<|tool_call_start|>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ },
43
+ "65511": {
44
+ "content": "<|tool_call_end|>",
45
+ "lstrip": false,
46
+ "normalized": false,
47
+ "rstrip": false,
48
+ "single_word": false,
49
+ "special": true
50
+ },
51
+ "65512": {
52
+ "content": "<|tool_list_start|>",
53
+ "lstrip": false,
54
+ "normalized": false,
55
+ "rstrip": false,
56
+ "single_word": false,
57
+ "special": true
58
+ },
59
+ "65513": {
60
+ "content": "<|tool_list_end|>",
61
+ "lstrip": false,
62
+ "normalized": false,
63
+ "rstrip": false,
64
+ "single_word": false,
65
+ "special": true
66
+ },
67
+ "65514": {
68
+ "content": "<|source_analysis_start|>",
69
+ "lstrip": false,
70
+ "normalized": false,
71
+ "rstrip": false,
72
+ "single_word": false,
73
+ "special": true
74
+ },
75
+ "65515": {
76
+ "content": "<|source_analysis_end|>",
77
+ "lstrip": false,
78
+ "normalized": false,
79
+ "rstrip": false,
80
+ "single_word": false,
81
+ "special": true
82
+ },
83
+ "65516": {
84
+ "content": "<|source_start|>",
85
+ "lstrip": false,
86
+ "normalized": false,
87
+ "rstrip": false,
88
+ "single_word": false,
89
+ "special": true
90
+ },
91
+ "65517": {
92
+ "content": "<|source_end|>",
93
+ "lstrip": false,
94
+ "normalized": false,
95
+ "rstrip": false,
96
+ "single_word": false,
97
+ "special": true
98
+ },
99
+ "65518": {
100
+ "content": "<|im_start|>",
101
+ "lstrip": false,
102
+ "normalized": false,
103
+ "rstrip": false,
104
+ "single_word": false,
105
+ "special": true
106
+ },
107
+ "65519": {
108
+ "content": "<|im_end|>",
109
+ "lstrip": false,
110
+ "normalized": false,
111
+ "rstrip": false,
112
+ "single_word": false,
113
+ "special": true
114
+ },
115
+ "65520": {
116
+ "content": "<|answer_start|>",
117
+ "lstrip": false,
118
+ "normalized": false,
119
+ "rstrip": false,
120
+ "single_word": false,
121
+ "special": true
122
+ },
123
+ "65521": {
124
+ "content": "<|answer_end|>",
125
+ "lstrip": false,
126
+ "normalized": false,
127
+ "rstrip": false,
128
+ "single_word": false,
129
+ "special": true
130
+ },
131
+ "65522": {
132
+ "content": "<|text_start|>",
133
+ "lstrip": false,
134
+ "normalized": false,
135
+ "rstrip": false,
136
+ "single_word": false,
137
+ "special": true
138
+ },
139
+ "65523": {
140
+ "content": "<|text_end|>",
141
+ "lstrip": false,
142
+ "normalized": false,
143
+ "rstrip": false,
144
+ "single_word": false,
145
+ "special": true
146
+ },
147
+ "65524": {
148
+ "content": "<|translation_start|>",
149
+ "lstrip": false,
150
+ "normalized": false,
151
+ "rstrip": false,
152
+ "single_word": false,
153
+ "special": true
154
+ },
155
+ "65525": {
156
+ "content": "<|translation_end|>",
157
+ "lstrip": false,
158
+ "normalized": false,
159
+ "rstrip": false,
160
+ "single_word": false,
161
+ "special": true
162
+ },
163
  "65526": {
164
+ "content": "<|back_translation_start|>",
165
  "lstrip": false,
166
  "normalized": false,
167
  "rstrip": false,
 
169
  "special": true
170
  },
171
  "65527": {
172
+ "content": "<|back_translation_end|>",
173
  "lstrip": false,
174
  "normalized": false,
175
  "rstrip": false,
 
177
  "special": true
178
  },
179
  "65528": {
180
+ "content": "<|ocr_correction_start|>",
181
  "lstrip": false,
182
  "normalized": false,
183
  "rstrip": false,
 
185
  "special": true
186
  },
187
  "65529": {
188
+ "content": "<|ocr_correction_end|>",
189
  "lstrip": false,
190
  "normalized": false,
191
  "rstrip": false,
 
193
  "special": true
194
  },
195
  "65530": {
196
+ "content": "<|json_scheme_start|>",
197
  "lstrip": false,
198
  "normalized": false,
199
  "rstrip": false,
 
201
  "special": true
202
  },
203
  "65531": {
204
+ "content": "<|json_scheme_end|>",
205
  "lstrip": false,
206
  "normalized": false,
207
  "rstrip": false,
 
209
  "special": true
210
  },
211
  "65532": {
212
+ "content": "<|source_interpretation_start|>",
213
  "lstrip": false,
214
  "normalized": false,
215
  "rstrip": false,
 
217
  "special": true
218
  },
219
  "65533": {
220
+ "content": "<|source_interpretation_end|>",
221
  "lstrip": false,
222
  "normalized": false,
223
  "rstrip": false,
 
225
  "special": true
226
  },
227
  "65534": {
228
+ "content": "<|query_start|>",
229
  "lstrip": false,
230
  "normalized": false,
231
  "rstrip": false,
 
233
  "special": true
234
  },
235
  "65535": {
236
+ "content": "<|query_end|>",
237
  "lstrip": false,
238
  "normalized": false,
239
  "rstrip": false,
 
242
  }
243
  },
244
  "additional_special_tokens": [
245
+ "<|tool_call_start|>",
246
+ "<|tool_call_end|>",
247
+ "<|tool_list_start|>",
248
+ "<|tool_list_end|>",
 
 
249
  "<|source_analysis_start|>",
250
  "<|source_analysis_end|>",
251
+ "<|source_start|>",
252
+ "<|source_end|>",
253
+ "<|im_start|>",
254
+ "<|im_end|>",
255
  "<|answer_start|>",
256
+ "<|answer_end|>",
257
+ "<|text_start|>",
258
+ "<|text_end|>",
259
+ "<|translation_start|>",
260
+ "<|translation_end|>",
261
+ "<|back_translation_start|>",
262
+ "<|back_translation_end|>",
263
+ "<|ocr_correction_start|>",
264
+ "<|ocr_correction_end|>",
265
+ "<|json_scheme_start|>",
266
+ "<|json_scheme_end|>",
267
+ "<|source_interpretation_start|>",
268
+ "<|source_interpretation_end|>",
269
+ "<|query_start|>",
270
+ "<|query_end|>"
271
  ],
272
  "clean_up_tokenization_spaces": true,
273
  "model_max_length": 1000000000000000019884624838656,