Orion-zhen commited on
Commit
3e4fb5d
1 Parent(s): c7d38ea

Add files using upload-large-folder tool

Browse files
config.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "LlamaForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "auto_map": {
8
+ "AutoTokenizer": [
9
+ "gptx_tokenizer.SPTokenizer",
10
+ null
11
+ ]
12
+ },
13
+ "bos_token_id": 1,
14
+ "eos_token_id": 2,
15
+ "head_dim": 128,
16
+ "hidden_act": "silu",
17
+ "hidden_size": 4096,
18
+ "initializer_range": 0.0158,
19
+ "intermediate_size": 13440,
20
+ "max_position_embeddings": 4096,
21
+ "mlp_bias": false,
22
+ "model_type": "llama",
23
+ "num_attention_heads": 32,
24
+ "num_hidden_layers": 32,
25
+ "num_key_value_heads": 2,
26
+ "pad_token_id": 3,
27
+ "pretraining_tp": 1,
28
+ "rms_norm_eps": 1e-05,
29
+ "rope_scaling": null,
30
+ "rope_theta": 10000.0,
31
+ "tie_word_embeddings": true,
32
+ "tokenizer_class": "SPTokenizer",
33
+ "torch_dtype": "bfloat16",
34
+ "transformers_version": "4.47.0",
35
+ "use_cache": true,
36
+ "vocab_size": 250680
37
+ }
generation_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": 2,
5
+ "pad_token_id": 3,
6
+ "transformers_version": "4.47.0"
7
+ }
gptx_tokenizer.py ADDED
@@ -0,0 +1,465 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ import os
5
+ import warnings
6
+ from pathlib import Path
7
+ from typing import Any, Dict, List, Mapping, Optional, Tuple, Union
8
+
9
+ import sentencepiece as spm
10
+ import numpy as np
11
+ import torch
12
+ from huggingface_hub import hf_hub_download, list_repo_files, try_to_load_from_cache
13
+ from transformers.tokenization_utils import PreTrainedTokenizer
14
+ from transformers.tokenization_utils_base import TOKENIZER_CONFIG_FILE
15
+
16
+
17
+ REPO_ID = "openGPT-X/Teuken-7B-instruct-research-v0.4"
18
+
19
+ class HFGPTXTokenizer(PreTrainedTokenizer):
20
+ """
21
+ A custom tokenizer class that extends Hugging Face's PreTrainedTokenizer.
22
+ It is specifically designed to work with SentencePiece models and integrates
23
+ with Hugging Face's tokenizer utilities.
24
+ """
25
+
26
+ model_file_glob = "*tokenizer.json"
27
+ vocab_files_names = {"tokenizer_file": "tokenizer.json"}
28
+ decode_kwargs: List[str] = []
29
+
30
+ def _encode(self, text: str, return_tokens: bool = False, is_continuation: bool = False):
31
+ """
32
+ Encode a given text using the tokenizer.
33
+
34
+ Args:
35
+ text (str): The text to encode.
36
+ return_tokens (bool): If True, returns token strings instead of token IDs.
37
+ is_continuation (bool): If True, uses a continuation tokenizer (if available).
38
+ Returns:
39
+ List[int] or List[str]: Encoded text as a list of token IDs or token strings.
40
+ """
41
+ assert self.tok is not None, "No tokenizer is currently loaded"
42
+
43
+ # Variant with additional sp processor:
44
+ tokenizer = self.continuation_tokenizer if is_continuation else self.tok
45
+
46
+ if return_tokens:
47
+ return tokenizer.encode_as_pieces(text)
48
+ else:
49
+ return tokenizer.encode(text)
50
+
51
+ def create_list_of_special_tokens(self) -> List[str]:
52
+ """
53
+ Create a list of special tokens, including the BOS, EOS, PAD, EOD tokens,
54
+ and 256 additional placeholder tokens.
55
+ Returns:
56
+ List[str]: List of special tokens.
57
+ """
58
+ return [self.bos_token, self.eos_token, self.pad_token, self.eod_token] + [
59
+ f"<placeholder_tok_{i}>" for i in range(256)
60
+ ]
61
+
62
+ def find_tokenizer_config(self, config_path: Path, repo_id: str = None) -> Optional[Path]:
63
+ if not os.path.isfile(config_path):
64
+ config_path = try_to_load_from_cache(repo_id=repo_id, filename=Path(config_path).name)
65
+ if not config_path:
66
+ config_path = self._download_config_from_hub(repo_id=repo_id)
67
+
68
+ return config_path
69
+
70
+
71
+ def instantiate_from_file_or_name(self, model_file_or_name: str, repo_id: str = None):
72
+ """
73
+ Load the tokenizer model from a file or download it from a repository.
74
+
75
+ Args:
76
+ model_file_or_name (str): Path to the model file or the model name.
77
+ repo_id (str, optional): Repository ID from which to download the model file.
78
+
79
+ Returns:
80
+ spm.SentencePieceProcessor: Loaded SentencePieceProcessor instance.
81
+
82
+ Raises:
83
+ ValueError: If repo_id is not provided when model_file_or_name is not a file.
84
+ OSError: If the model file cannot be loaded or downloaded.
85
+ """
86
+ if not os.path.isfile(model_file_or_name):
87
+ model_file_or_name = try_to_load_from_cache(repo_id=repo_id, filename=Path(model_file_or_name).name)
88
+ if not model_file_or_name:
89
+ model_file_or_name = self._download_model_from_hub(repo_id=repo_id)
90
+
91
+ try:
92
+ return spm.SentencePieceProcessor(model_file=model_file_or_name)
93
+ except Exception as e:
94
+ raise OSError(f"Failed to load tokenizer model: {str(e)}")
95
+
96
+ def _download_model_from_hub(self, repo_id: str) -> Optional[str]:
97
+ try:
98
+ # List all files in the repo
99
+ repo_files = list_repo_files(repo_id)
100
+
101
+ # Find the tokenizer model file
102
+ tokenizer_files = [f for f in repo_files if f.endswith('.model')]
103
+ if not tokenizer_files:
104
+ raise FileNotFoundError(f"No .model file found in repository {repo_id}")
105
+
106
+ # Use the first .model file found
107
+ model_file = tokenizer_files[0]
108
+ print(f"Found tokenizer model file: {model_file}")
109
+
110
+ # Download the file
111
+ model_file_or_name = hf_hub_download(repo_id=repo_id, filename=model_file)
112
+ print(f"Downloaded tokenizer model to: {model_file_or_name}")
113
+ except Exception as e:
114
+ raise OSError(f"Failed to download tokenizer model: {str(e)}")
115
+
116
+ return model_file_or_name
117
+
118
+ def _download_config_from_hub(self, repo_id: str):
119
+ if repo_id is None:
120
+ raise ValueError("repo_id must be provided if config_path is not a local file")
121
+
122
+ try:
123
+ # List all files in the repo
124
+ repo_files = list_repo_files(repo_id)
125
+
126
+ # Find the tokenizer config file
127
+ tokenizer_files = [f for f in repo_files if f.endswith('tokenizer_config.json')]
128
+ if not tokenizer_files:
129
+ raise FileNotFoundError(f"No tokenizer_config.json file found in repository {repo_id}")
130
+
131
+ # Use the first tokenizer_config.json file found
132
+ tokenizer_config_file = tokenizer_files[0]
133
+ print(f"Found tokenizer config file: {tokenizer_config_file}")
134
+
135
+ # Download the file
136
+ tokenizer_config_file_or_name = hf_hub_download(repo_id=repo_id, filename=tokenizer_config_file)
137
+ print(f"Downloaded tokenizer config file to: {tokenizer_config_file_or_name}")
138
+ return tokenizer_config_file_or_name
139
+ except Exception as e:
140
+ raise OSError(f"Failed to download tokenizer model: {str(e)}")
141
+ def __init__(
142
+ self,
143
+ model_path: Optional[str] = None,
144
+ config_path: Optional[str] = None,
145
+ **kwargs: Any,
146
+ ) -> None:
147
+ """
148
+ Initialize the tokenizer.
149
+ Args:
150
+ model_path (Optional[str]): Path to the tokenizer model file.
151
+ config_path (Optional[str]): Path to the tokenizer configuration file.
152
+ **kwargs: Additional keyword arguments passed to the superclass.
153
+ This method also ensures backward compatibility by setting
154
+ `clean_up_tokenization_spaces` to False by default.
155
+ """
156
+ # Prevent cleanup of tokenization spaces to maintain backward compatibility
157
+ self.clean_up_tokenization_spaces = kwargs.setdefault("clean_up_tokenization_spaces", False)
158
+ self.vocab = None
159
+ cp_path = kwargs.get("name_or_path", ".")
160
+ if model_path is None:
161
+ model_path = str(Path(cp_path) / self.vocab_files_names["tokenizer_file"])
162
+ self.tok = self.instantiate_from_file_or_name(model_path, repo_id=REPO_ID)
163
+
164
+ super().__init__(**kwargs)
165
+
166
+ # Specify special tokens which we know the value of.
167
+ # EOD from `tok` is used as what is called EOS in HuggingFace.
168
+ # Since there is no corresponding mapping for EOS from `tok` in
169
+ # HuggingFace, it is treated as an additional special token.
170
+ # Same for all other special tokens.
171
+
172
+
173
+ self.unk_token = "<unk>"
174
+ self.eos_token = "</s>"
175
+ self.bos_token = "<s>"
176
+ self.pad_token = "<pad>"
177
+ self.eod_token = "<eod>"
178
+
179
+ self.additional_special_tokens = self.create_list_of_special_tokens()
180
+
181
+ if config_path is None:
182
+ config_path = str(Path(cp_path) / TOKENIZER_CONFIG_FILE)
183
+
184
+ if os.path.isfile(config_path):
185
+ self.tokenizer_config = self.load_json(Path(config_path))
186
+ else: # Load from repo
187
+ self.tokenizer_config = self.load_json(Path(self.find_tokenizer_config(Path(config_path), repo_id=REPO_ID)))
188
+
189
+ @property
190
+ def vocab_size(self) -> int:
191
+ """
192
+ Get the size of the tokenizer vocabulary.
193
+ Returns:
194
+ int: The size of the vocabulary.
195
+ """
196
+ return self.tok.GetPieceSize()
197
+
198
+ def get_vocab(self) -> Dict[str, int]:
199
+ """
200
+ Get the vocabulary as a dictionary mapping token strings to their IDs.
201
+ Returns:
202
+ Dict[str, int]: Vocabulary mapping.
203
+ """
204
+ if self.vocab is None:
205
+ self.vocab = {self.tok.IdToPiece(i): i for i in range(self.vocab_size)}
206
+ return self.vocab
207
+
208
+ def _tokenize(self, text: str, **kwargs) -> List[int]:
209
+ """
210
+ Tokenize the input text.
211
+ Args:
212
+ text (str): Text to tokenize.
213
+ **kwargs: Additional keyword arguments.
214
+ Returns:
215
+ List[int]: List of token IDs.
216
+ """
217
+ return_tokens = kwargs.pop("return_tokens", True)
218
+ return self._encode(text, return_tokens=return_tokens, **kwargs)
219
+
220
+ def _convert_token_to_id(self, token: str) -> int:
221
+ """
222
+ Convert a token string to its corresponding ID.
223
+ Args:
224
+ token (str): The token to convert.
225
+ Returns:
226
+ int: The token's ID.
227
+ Raises:
228
+ ValueError: If the token is unknown and cannot be encoded to a single ID.
229
+ """
230
+ return self.tok.PieceToId(token)
231
+
232
+
233
+ def decode(
234
+ self,
235
+ token_ids: Union[List[int], List[List[int]]],
236
+ num_threads: Optional[int] = None,
237
+ skip_special_tokens: bool = False,
238
+ clean_up_tokenization_spaces: bool = False,
239
+ ) -> str:
240
+ """
241
+ Decode a list of token IDs into a string.
242
+ Args:
243
+ token_ids (Union[List[int], List[List[int]]]): List of token IDs or lists of token IDs.
244
+ num_threads (Optional[int]): Number of threads to use for decoding.
245
+ Returns:
246
+ str: Decoded string.
247
+ """
248
+ if isinstance(token_ids, torch.Tensor): # For PyTorch tensors
249
+ token_ids = token_ids.tolist()
250
+ elif isinstance(token_ids, np.ndarray): # For NumPy arrays
251
+ token_ids = token_ids.tolist()
252
+
253
+ output = self.tok.decode(input=token_ids, num_threads=num_threads)
254
+ if skip_special_tokens:
255
+ for substring in self.additional_special_tokens:
256
+ output = output.replace(substring, "")
257
+
258
+ if clean_up_tokenization_spaces:
259
+ warnings.warn(
260
+ "when cleaning up tokenization spaces, this will not behave "
261
+ "like the original `GPTXTokenizer`., Please supply "
262
+ "`clean_up_tokenization_spaces=False` for decoding."
263
+ )
264
+ output = self.clean_up_tokenization(output)
265
+
266
+ return output
267
+
268
+
269
+ def _convert_id_to_token(self, index: int) -> str:
270
+ """
271
+ Convert a token ID to its corresponding token string.
272
+ Args:
273
+ index (int): Token ID.
274
+ Returns:
275
+ str: Corresponding token string.
276
+ """
277
+ return self.tok.IdToPiece(index)
278
+
279
+ def convert_tokens_to_string(self, tokens: List[str]) -> str:
280
+ """
281
+ Convert a list of tokens into a single string.
282
+ Args:
283
+ tokens (List[str]): List of token strings.
284
+ Returns:
285
+ str: Concatenated string of tokens.
286
+ """
287
+ return self.tok.DecodePieces(tokens)
288
+
289
+ def _tok_decode(self, token_ids: List[int], **kwargs: Any) -> str:
290
+ """
291
+ Internal method to decode token IDs with additional arguments.
292
+ Args:
293
+ token_ids (List[int]): List of token IDs.
294
+ **kwargs: Additional arguments to pass to the decode method.
295
+ Returns:
296
+ str: Decoded string.
297
+ This method also issues a warning if unsupported arguments are provided.
298
+ """
299
+ passed_kwargs = {key: value for (key, value) in kwargs.items() if key in self.decode_kwargs}
300
+ if len(passed_kwargs) != len(kwargs):
301
+ warnings.warn("silently ignoring some arguments to `decode` due to missing " "support from the tokenizer.")
302
+ text = self.decode(token_ids, **passed_kwargs)
303
+ return text
304
+
305
+ def save_tokenizer(self, save_dir: str) -> None:
306
+ if not os.path.isdir(save_dir):
307
+ print(f"Vocabulary path ({save_dir}) should be a directory")
308
+ return
309
+ out_vocab_file = os.path.join(save_dir, "tokenizer.model")
310
+
311
+ # if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
312
+ # copyfile(self.vocab_file, out_vocab_file)
313
+ # elif not os.path.isfile(self.vocab_file):
314
+ with open(out_vocab_file, "wb") as f:
315
+ content_spiece_model = self.tok.serialized_model_proto()
316
+ f.write(content_spiece_model)
317
+
318
+ return (out_vocab_file,)
319
+
320
+ def _decode(
321
+ self,
322
+ token_ids: List[int],
323
+ skip_special_tokens: bool = False,
324
+ clean_up_tokenization_spaces: bool = None,
325
+ spaces_between_special_tokens: bool = True,
326
+ **kwargs: Any,
327
+ ) -> str:
328
+ text = self._tok_decode(
329
+ token_ids,
330
+ skip_special_tokens=skip_special_tokens,
331
+ spaces_between_special_tokens=spaces_between_special_tokens,
332
+ **kwargs,
333
+ )
334
+
335
+ clean_up_tokenization_spaces = (
336
+ clean_up_tokenization_spaces
337
+ if clean_up_tokenization_spaces is not None
338
+ else self.clean_up_tokenization_spaces
339
+ )
340
+ if clean_up_tokenization_spaces:
341
+ warnings.warn(
342
+ "when cleaning up tokenization spaces, this will not behave "
343
+ "like the original `GPTXTokenizer`., Please supply "
344
+ "`clean_up_tokenization_spaces=False` for decoding."
345
+ )
346
+ clean_text = self.clean_up_tokenization(text)
347
+ return clean_text
348
+ else:
349
+ return text
350
+
351
+ def save_vocabulary(
352
+ self,
353
+ save_directory: str,
354
+ filename_prefix: Optional[str] = None,
355
+ ) -> Tuple[str]:
356
+ filename_prefix = filename_prefix + "-" if filename_prefix else ""
357
+ save_directory = Path(save_directory)
358
+
359
+ self._save_tokenizer_config(save_directory, filename_prefix)
360
+ tokenizer_file_path = self._save_tokenizer(save_directory, filename_prefix)
361
+
362
+ return (tokenizer_file_path,)
363
+
364
+ def _save_tokenizer_config(
365
+ self,
366
+ save_directory: Path,
367
+ filename_prefix: str,
368
+ ) -> str:
369
+ self.save_tokenizer_config(save_directory)
370
+ old_tokenizer_config_path = save_directory / TOKENIZER_CONFIG_FILE
371
+ assert old_tokenizer_config_path.is_file(), "tokenizer config path changed"
372
+ new_tokenizer_config_path = save_directory / (filename_prefix + old_tokenizer_config_path.name)
373
+ old_tokenizer_config_path.replace(new_tokenizer_config_path)
374
+ return str(new_tokenizer_config_path)
375
+
376
+ def _find_tokenizer_files(self, save_directory: Path) -> List[Path]:
377
+ files = list(Path(save_directory).glob(self.model_file_glob))
378
+ return files
379
+
380
+ def _get_tokenizer_file(self, files: List[Path]):
381
+ assert files, "no saved tokenizer file found"
382
+ assert len(files) <= 1, "cannot handle multiple saved tokenizer files"
383
+ return files[0]
384
+
385
+ def _save_tokenizer(
386
+ self,
387
+ save_directory: Path,
388
+ filename_prefix: str,
389
+ ) -> str:
390
+ self.save_tokenizer(str(save_directory))
391
+ tokenizer_files = self._find_tokenizer_files(save_directory)
392
+ old_tokenizer_file_path = self._get_tokenizer_file(tokenizer_files)
393
+ assert old_tokenizer_file_path.is_file(), "could not access saved tokenizer file"
394
+ new_tokenizer_file_path = save_directory / (filename_prefix + self.vocab_files_names["tokenizer_file"])
395
+ old_tokenizer_file_path.replace(new_tokenizer_file_path)
396
+ return str(new_tokenizer_file_path)
397
+
398
+ def save_tokenizer_config(self, save_dir: Union[str, Path]) -> None:
399
+ save_dir = Path(save_dir)
400
+
401
+ # convert Path to str
402
+ for k in self.tokenizer_config:
403
+ if isinstance(self.tokenizer_config[k], Path):
404
+ self.tokenizer_config[k] = str(self.tokenizer_config[k])
405
+
406
+ info_file = save_dir / "tokenizer_config.json"
407
+ with info_file.open("w") as f:
408
+ json.dump(self.tokenizer_config, f, indent=4)
409
+
410
+ def load_json(self, path: Path) -> dict:
411
+ with path.open("r") as f:
412
+ return json.load(f)
413
+
414
+ class SPTokenizer(HFGPTXTokenizer):
415
+ model_file_glob = "*tokenizer.model"
416
+ vocab_files_names = {"tokenizer_file": "tokenizer.model"}
417
+ decode_kwargs = ["num_threads"]
418
+ # `is_continuation` does not work without this, but it doesn't
419
+ # implement all APIs of `PreTrainedTokenizer`.
420
+ def encode(self, text: str, **kwargs) -> List[int]:
421
+ return_tokens = kwargs.pop('return_tokens', False)
422
+ is_continuation = kwargs.pop('is_continuation', False)
423
+ return self._encode(
424
+ text,
425
+ return_tokens=return_tokens,
426
+ is_continuation=is_continuation,
427
+ )
428
+
429
+ def __init__(self, *args, **kwargs):
430
+ super().__init__(*args, **kwargs)
431
+
432
+ self.eos_token = "</s>"
433
+ self.eos_token_id = 2
434
+ self.system_messages_by_lang = { # translations by deepl / google translate
435
+ "BG": "Чат между човек и асистент с изкуствен интелект. Асистентът дава полезни и учтиви отговори на въпросите на човека.", # noqa
436
+ "CS": "Chat mezi člověkem a asistentem s umělou inteligencí. Asistent poskytuje vstřícné a zdvořilé odpovědi na otázky člověka.", # noqa
437
+ "DA": "En chat mellem et menneske og en assistent med kunstig intelligens, som giver hjælpsomme og høflige svar på menneskets spørgsmål.", # noqa
438
+ "DE": "Ein Gespräch zwischen einem Menschen und einem Assistenten mit künstlicher Intelligenz. Der Assistent gibt hilfreiche und höfliche Antworten auf die Fragen des Menschen.", # noqa
439
+ "EL": "Μια συνομιλία μεταξύ ενός ανθρώπου και ενός βοηθού τεχνητής νοημοσύνης. Ο βοηθός δίνει χρήσιμες και ευγενικές απαντήσεις στις ερωτήσεις του ανθρώπου.", # noqa
440
+ "EN": "A chat between a human and an artificial intelligence assistant.The assistant gives helpful and polite answers to the human's questions.", # noqa
441
+ "ES": "Una conversación entre un humano y un asistente de inteligencia artificial. El asistente da respuestas útiles y amables a las preguntas del humano.", # noqa
442
+ "ET": "Inimese ja tehisintellekti assistendi vaheline vestlus. Assistent annab inimese küsimustele abivalmis ja viisakaid vastuseid.", # noqa
443
+ "FI": "Ihmisen ja tekoälyavustajan välinen keskustelu. Avustaja antaa avuliaita ja kohteliaita vastauksia ihmisen kysymyksiin.", # noqa
444
+ "FR": "Conversation entre un humain et un assistant doté d'une intelligence artificielle. L'assistant donne des réponses utiles et polies aux questions de l'homme.", # noqa
445
+ "GA": "Comhrá idir duine agus cúntóir hintleachta saorga. Tugann an cúntóir freagraí cabhracha dea-bhéasacha ar cheisteanna an duine.", # noqa
446
+ "HR": "Razgovor između čovjeka i pomoćnika umjetne inteligencije. Pomoćnik daje korisne i ljubazne odgovore na ljudska pitanja.", # noqa
447
+ "HU": "Egy ember és egy mesterséges intelligencia asszisztens közötti beszélgetés. Az asszisztens segítőkész és udvarias válaszokat ad az ember kérdéseire.", # noqa
448
+ "IT": "Una chat tra un umano e un assistente di intelligenza artificiale. L'assistente fornisce risposte utili ed educate alle domande dell'uomo.", # noqa
449
+ "LT": "Žmogaus ir dirbtinio intelekto asistento pokalbis. Asistentas naudingai ir mandagiai atsako į žmogaus klausimus.", # noqa
450
+ "LV": "Cilvēka un mākslīgā intelekta asistenta tērzēšana. Asistents sniedz noderīgas un pieklājīgas atbildes uz cilvēka jautājumiem.", # noqa
451
+ "MT": "Chat bejn bniedem u assistent ta' intelliġenza artifiċjali. L-assistent jagħti tweġibiet ta' għajnuna u edukat għall-mistoqsijiet tal-bniedem.", # noqa
452
+ "NL": "Een chat tussen een mens en een assistent met kunstmatige intelligentie. De assistent geeft behulpzame en beleefde antwoorden op de vragen van de mens.", # noqa
453
+ "PL": "Czat między człowiekiem a asystentem sztucznej inteligencji. Asystent udziela pomocnych i uprzejmych odpowiedzi na pytania człowieka.", # noqa
454
+ "PT": "Uma conversa entre um ser humano e um assistente de inteligência artificial. O assistente dá respostas úteis e educadas às perguntas do utilizador.", # noqa
455
+ "RO": "O conversație între un om și un asistent cu inteligență artificială. Asistentul oferă răspunsuri utile și politicoase la întrebările omului.", # noqa
456
+ "SK": "Rozhovor medzi človekom a asistentom s umelou inteligenciou. Asistent poskytuje užitočné a zdvorilé odpovede na otázky človeka.", # noqa
457
+ "SL": "Pogovor med človekom in pomočnikom z umetno inteligenco. Pomočnik človeku prijazno in vljudno odgovarja na njegova vprašanja.", # noqa
458
+ "SV": "En chatt mellan en människa och en assistent med artificiell intelligens. Assistenten ger hjälpsamma och artiga svar på människans frågor.", # noqa
459
+ }
460
+ chat_template = "{%- for message in messages %}\n{%- if (message['role']|lower == 'user') != (loop.index0 % 2 == 0) %}\n{{- raise_exception('Roles must alternate User/Assistant/User/Assistant/...') }}\n{%- endif %}\n{%-if message['role']|lower == 'user' %}\n{{- message['role']|capitalize + ': ' + message['content'] + '\\n' }}\n{%- elif message['role']|lower == 'assistant' %}\n{{- message['role']|capitalize + ': ' + message['content'] + eos_token + '\\n' }}\n{%- else %}\n{{- raise_exception('Only user and assistant roles are supported!') }}\n {%- endif %}\n{%- endfor %}{%-if add_generation_prompt %}\n{{- 'Assistant: '}}\n{%- endif %}\n"
461
+ self.chat_template = {
462
+ lang: f"System: {sys_msg}" + "{{- '\\n'}}\n" + chat_template
463
+ for lang, sys_msg in self.system_messages_by_lang.items()
464
+ }
465
+ self.chat_template['default'] = f"System: {self.system_messages_by_lang['EN']}" + "{{- '\\n'}}\n" + chat_template
model-00001-of-00004.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2d7d4006a16e1148cab1fa7dbb3647fbf0aeb42a83b5e9aa7599c900cd308194
3
+ size 4936228544
model-00002-of-00004.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c65ef774c48565b10781bff7d115ef07fc1473147680ca922f2c4ebf265e715e
3
+ size 4929565016
model-00003-of-00004.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9467cd2460a427aa26654165a204fe2f1250cba4806b5931552c765ed28e4aeb
3
+ size 4929565032
model-00004-of-00004.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:24763318a05c7f049cf4aadb8ef78edaf6740fa6636a0a81018d394f01135adf
3
+ size 110125496
model.safetensors.index.json ADDED
@@ -0,0 +1,297 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "metadata": {
3
+ "total_size": 14905450496
4
+ },
5
+ "weight_map": {
6
+ "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
7
+ "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
8
+ "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
9
+ "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
10
+ "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
11
+ "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
12
+ "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
13
+ "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
14
+ "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
15
+ "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
16
+ "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
17
+ "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
18
+ "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
19
+ "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
20
+ "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
21
+ "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
22
+ "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
23
+ "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
24
+ "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
25
+ "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
26
+ "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
27
+ "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
28
+ "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
29
+ "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
30
+ "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
31
+ "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
32
+ "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
33
+ "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
34
+ "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
35
+ "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
36
+ "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
37
+ "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
38
+ "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
39
+ "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
40
+ "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
41
+ "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
42
+ "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
43
+ "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
44
+ "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
45
+ "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
46
+ "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
47
+ "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
48
+ "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
49
+ "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
50
+ "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
51
+ "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
52
+ "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
53
+ "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
54
+ "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
55
+ "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
56
+ "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
57
+ "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
58
+ "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
59
+ "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
60
+ "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
61
+ "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
62
+ "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
63
+ "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
64
+ "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
65
+ "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
66
+ "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
67
+ "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
68
+ "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
69
+ "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
70
+ "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
71
+ "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
72
+ "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
73
+ "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
74
+ "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
75
+ "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
76
+ "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
77
+ "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
78
+ "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
79
+ "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
80
+ "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
81
+ "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
82
+ "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
83
+ "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
84
+ "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
85
+ "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
86
+ "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
87
+ "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
88
+ "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
89
+ "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
90
+ "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
91
+ "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
92
+ "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
93
+ "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
94
+ "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
95
+ "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
96
+ "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
97
+ "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
98
+ "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
99
+ "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
100
+ "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
101
+ "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
102
+ "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
103
+ "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
104
+ "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
105
+ "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
106
+ "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
107
+ "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
108
+ "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
109
+ "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
110
+ "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
111
+ "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
112
+ "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
113
+ "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
114
+ "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
115
+ "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
116
+ "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
117
+ "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
118
+ "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
119
+ "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
120
+ "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
121
+ "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
122
+ "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
123
+ "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
124
+ "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
125
+ "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
126
+ "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
127
+ "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
128
+ "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
129
+ "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
130
+ "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
131
+ "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
132
+ "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
133
+ "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
134
+ "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
135
+ "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
136
+ "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
137
+ "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
138
+ "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
139
+ "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
140
+ "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
141
+ "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
142
+ "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
143
+ "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
144
+ "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
145
+ "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
146
+ "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
147
+ "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
148
+ "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
149
+ "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
150
+ "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
151
+ "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
152
+ "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
153
+ "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
154
+ "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
155
+ "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
156
+ "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
157
+ "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
158
+ "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
159
+ "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
160
+ "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
161
+ "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
162
+ "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
163
+ "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
164
+ "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
165
+ "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
166
+ "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
167
+ "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
168
+ "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
169
+ "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
170
+ "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
171
+ "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
172
+ "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
173
+ "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
174
+ "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
175
+ "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
176
+ "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
177
+ "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
178
+ "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
179
+ "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
180
+ "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
181
+ "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
182
+ "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
183
+ "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
184
+ "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
185
+ "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
186
+ "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
187
+ "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
188
+ "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
189
+ "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
190
+ "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
191
+ "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
192
+ "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
193
+ "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
194
+ "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
195
+ "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
196
+ "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
197
+ "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
198
+ "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
199
+ "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
200
+ "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
201
+ "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
202
+ "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
203
+ "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
204
+ "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
205
+ "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
206
+ "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
207
+ "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
208
+ "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
209
+ "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
210
+ "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
211
+ "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
212
+ "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
213
+ "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
214
+ "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
215
+ "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
216
+ "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
217
+ "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
218
+ "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
219
+ "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
220
+ "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
221
+ "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
222
+ "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
223
+ "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
224
+ "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
225
+ "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
226
+ "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
227
+ "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
228
+ "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
229
+ "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
230
+ "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
231
+ "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
232
+ "model.layers.31.input_layernorm.weight": "model-00004-of-00004.safetensors",
233
+ "model.layers.31.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
234
+ "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
235
+ "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
236
+ "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
237
+ "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
238
+ "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
239
+ "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
240
+ "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
241
+ "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
242
+ "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
243
+ "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
244
+ "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
245
+ "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
246
+ "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
247
+ "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
248
+ "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
249
+ "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
250
+ "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
251
+ "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
252
+ "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
253
+ "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
254
+ "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
255
+ "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
256
+ "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
257
+ "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
258
+ "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
259
+ "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
260
+ "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
261
+ "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
262
+ "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
263
+ "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
264
+ "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
265
+ "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
266
+ "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
267
+ "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
268
+ "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
269
+ "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
270
+ "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
271
+ "model.layers.7.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
272
+ "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
273
+ "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
274
+ "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
275
+ "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
276
+ "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
277
+ "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
278
+ "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
279
+ "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
280
+ "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
281
+ "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
282
+ "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
283
+ "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
284
+ "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
285
+ "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
286
+ "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
287
+ "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
288
+ "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
289
+ "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
290
+ "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
291
+ "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
292
+ "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
293
+ "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
294
+ "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
295
+ "model.norm.weight": "model-00004-of-00004.safetensors"
296
+ }
297
+ }
special_tokens_map.json ADDED
@@ -0,0 +1,268 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<s>",
4
+ "</s>",
5
+ "<pad>",
6
+ "<eod>",
7
+ "<placeholder_tok_0>",
8
+ "<placeholder_tok_1>",
9
+ "<placeholder_tok_2>",
10
+ "<placeholder_tok_3>",
11
+ "<placeholder_tok_4>",
12
+ "<placeholder_tok_5>",
13
+ "<placeholder_tok_6>",
14
+ "<placeholder_tok_7>",
15
+ "<placeholder_tok_8>",
16
+ "<placeholder_tok_9>",
17
+ "<placeholder_tok_10>",
18
+ "<placeholder_tok_11>",
19
+ "<placeholder_tok_12>",
20
+ "<placeholder_tok_13>",
21
+ "<placeholder_tok_14>",
22
+ "<placeholder_tok_15>",
23
+ "<placeholder_tok_16>",
24
+ "<placeholder_tok_17>",
25
+ "<placeholder_tok_18>",
26
+ "<placeholder_tok_19>",
27
+ "<placeholder_tok_20>",
28
+ "<placeholder_tok_21>",
29
+ "<placeholder_tok_22>",
30
+ "<placeholder_tok_23>",
31
+ "<placeholder_tok_24>",
32
+ "<placeholder_tok_25>",
33
+ "<placeholder_tok_26>",
34
+ "<placeholder_tok_27>",
35
+ "<placeholder_tok_28>",
36
+ "<placeholder_tok_29>",
37
+ "<placeholder_tok_30>",
38
+ "<placeholder_tok_31>",
39
+ "<placeholder_tok_32>",
40
+ "<placeholder_tok_33>",
41
+ "<placeholder_tok_34>",
42
+ "<placeholder_tok_35>",
43
+ "<placeholder_tok_36>",
44
+ "<placeholder_tok_37>",
45
+ "<placeholder_tok_38>",
46
+ "<placeholder_tok_39>",
47
+ "<placeholder_tok_40>",
48
+ "<placeholder_tok_41>",
49
+ "<placeholder_tok_42>",
50
+ "<placeholder_tok_43>",
51
+ "<placeholder_tok_44>",
52
+ "<placeholder_tok_45>",
53
+ "<placeholder_tok_46>",
54
+ "<placeholder_tok_47>",
55
+ "<placeholder_tok_48>",
56
+ "<placeholder_tok_49>",
57
+ "<placeholder_tok_50>",
58
+ "<placeholder_tok_51>",
59
+ "<placeholder_tok_52>",
60
+ "<placeholder_tok_53>",
61
+ "<placeholder_tok_54>",
62
+ "<placeholder_tok_55>",
63
+ "<placeholder_tok_56>",
64
+ "<placeholder_tok_57>",
65
+ "<placeholder_tok_58>",
66
+ "<placeholder_tok_59>",
67
+ "<placeholder_tok_60>",
68
+ "<placeholder_tok_61>",
69
+ "<placeholder_tok_62>",
70
+ "<placeholder_tok_63>",
71
+ "<placeholder_tok_64>",
72
+ "<placeholder_tok_65>",
73
+ "<placeholder_tok_66>",
74
+ "<placeholder_tok_67>",
75
+ "<placeholder_tok_68>",
76
+ "<placeholder_tok_69>",
77
+ "<placeholder_tok_70>",
78
+ "<placeholder_tok_71>",
79
+ "<placeholder_tok_72>",
80
+ "<placeholder_tok_73>",
81
+ "<placeholder_tok_74>",
82
+ "<placeholder_tok_75>",
83
+ "<placeholder_tok_76>",
84
+ "<placeholder_tok_77>",
85
+ "<placeholder_tok_78>",
86
+ "<placeholder_tok_79>",
87
+ "<placeholder_tok_80>",
88
+ "<placeholder_tok_81>",
89
+ "<placeholder_tok_82>",
90
+ "<placeholder_tok_83>",
91
+ "<placeholder_tok_84>",
92
+ "<placeholder_tok_85>",
93
+ "<placeholder_tok_86>",
94
+ "<placeholder_tok_87>",
95
+ "<placeholder_tok_88>",
96
+ "<placeholder_tok_89>",
97
+ "<placeholder_tok_90>",
98
+ "<placeholder_tok_91>",
99
+ "<placeholder_tok_92>",
100
+ "<placeholder_tok_93>",
101
+ "<placeholder_tok_94>",
102
+ "<placeholder_tok_95>",
103
+ "<placeholder_tok_96>",
104
+ "<placeholder_tok_97>",
105
+ "<placeholder_tok_98>",
106
+ "<placeholder_tok_99>",
107
+ "<placeholder_tok_100>",
108
+ "<placeholder_tok_101>",
109
+ "<placeholder_tok_102>",
110
+ "<placeholder_tok_103>",
111
+ "<placeholder_tok_104>",
112
+ "<placeholder_tok_105>",
113
+ "<placeholder_tok_106>",
114
+ "<placeholder_tok_107>",
115
+ "<placeholder_tok_108>",
116
+ "<placeholder_tok_109>",
117
+ "<placeholder_tok_110>",
118
+ "<placeholder_tok_111>",
119
+ "<placeholder_tok_112>",
120
+ "<placeholder_tok_113>",
121
+ "<placeholder_tok_114>",
122
+ "<placeholder_tok_115>",
123
+ "<placeholder_tok_116>",
124
+ "<placeholder_tok_117>",
125
+ "<placeholder_tok_118>",
126
+ "<placeholder_tok_119>",
127
+ "<placeholder_tok_120>",
128
+ "<placeholder_tok_121>",
129
+ "<placeholder_tok_122>",
130
+ "<placeholder_tok_123>",
131
+ "<placeholder_tok_124>",
132
+ "<placeholder_tok_125>",
133
+ "<placeholder_tok_126>",
134
+ "<placeholder_tok_127>",
135
+ "<placeholder_tok_128>",
136
+ "<placeholder_tok_129>",
137
+ "<placeholder_tok_130>",
138
+ "<placeholder_tok_131>",
139
+ "<placeholder_tok_132>",
140
+ "<placeholder_tok_133>",
141
+ "<placeholder_tok_134>",
142
+ "<placeholder_tok_135>",
143
+ "<placeholder_tok_136>",
144
+ "<placeholder_tok_137>",
145
+ "<placeholder_tok_138>",
146
+ "<placeholder_tok_139>",
147
+ "<placeholder_tok_140>",
148
+ "<placeholder_tok_141>",
149
+ "<placeholder_tok_142>",
150
+ "<placeholder_tok_143>",
151
+ "<placeholder_tok_144>",
152
+ "<placeholder_tok_145>",
153
+ "<placeholder_tok_146>",
154
+ "<placeholder_tok_147>",
155
+ "<placeholder_tok_148>",
156
+ "<placeholder_tok_149>",
157
+ "<placeholder_tok_150>",
158
+ "<placeholder_tok_151>",
159
+ "<placeholder_tok_152>",
160
+ "<placeholder_tok_153>",
161
+ "<placeholder_tok_154>",
162
+ "<placeholder_tok_155>",
163
+ "<placeholder_tok_156>",
164
+ "<placeholder_tok_157>",
165
+ "<placeholder_tok_158>",
166
+ "<placeholder_tok_159>",
167
+ "<placeholder_tok_160>",
168
+ "<placeholder_tok_161>",
169
+ "<placeholder_tok_162>",
170
+ "<placeholder_tok_163>",
171
+ "<placeholder_tok_164>",
172
+ "<placeholder_tok_165>",
173
+ "<placeholder_tok_166>",
174
+ "<placeholder_tok_167>",
175
+ "<placeholder_tok_168>",
176
+ "<placeholder_tok_169>",
177
+ "<placeholder_tok_170>",
178
+ "<placeholder_tok_171>",
179
+ "<placeholder_tok_172>",
180
+ "<placeholder_tok_173>",
181
+ "<placeholder_tok_174>",
182
+ "<placeholder_tok_175>",
183
+ "<placeholder_tok_176>",
184
+ "<placeholder_tok_177>",
185
+ "<placeholder_tok_178>",
186
+ "<placeholder_tok_179>",
187
+ "<placeholder_tok_180>",
188
+ "<placeholder_tok_181>",
189
+ "<placeholder_tok_182>",
190
+ "<placeholder_tok_183>",
191
+ "<placeholder_tok_184>",
192
+ "<placeholder_tok_185>",
193
+ "<placeholder_tok_186>",
194
+ "<placeholder_tok_187>",
195
+ "<placeholder_tok_188>",
196
+ "<placeholder_tok_189>",
197
+ "<placeholder_tok_190>",
198
+ "<placeholder_tok_191>",
199
+ "<placeholder_tok_192>",
200
+ "<placeholder_tok_193>",
201
+ "<placeholder_tok_194>",
202
+ "<placeholder_tok_195>",
203
+ "<placeholder_tok_196>",
204
+ "<placeholder_tok_197>",
205
+ "<placeholder_tok_198>",
206
+ "<placeholder_tok_199>",
207
+ "<placeholder_tok_200>",
208
+ "<placeholder_tok_201>",
209
+ "<placeholder_tok_202>",
210
+ "<placeholder_tok_203>",
211
+ "<placeholder_tok_204>",
212
+ "<placeholder_tok_205>",
213
+ "<placeholder_tok_206>",
214
+ "<placeholder_tok_207>",
215
+ "<placeholder_tok_208>",
216
+ "<placeholder_tok_209>",
217
+ "<placeholder_tok_210>",
218
+ "<placeholder_tok_211>",
219
+ "<placeholder_tok_212>",
220
+ "<placeholder_tok_213>",
221
+ "<placeholder_tok_214>",
222
+ "<placeholder_tok_215>",
223
+ "<placeholder_tok_216>",
224
+ "<placeholder_tok_217>",
225
+ "<placeholder_tok_218>",
226
+ "<placeholder_tok_219>",
227
+ "<placeholder_tok_220>",
228
+ "<placeholder_tok_221>",
229
+ "<placeholder_tok_222>",
230
+ "<placeholder_tok_223>",
231
+ "<placeholder_tok_224>",
232
+ "<placeholder_tok_225>",
233
+ "<placeholder_tok_226>",
234
+ "<placeholder_tok_227>",
235
+ "<placeholder_tok_228>",
236
+ "<placeholder_tok_229>",
237
+ "<placeholder_tok_230>",
238
+ "<placeholder_tok_231>",
239
+ "<placeholder_tok_232>",
240
+ "<placeholder_tok_233>",
241
+ "<placeholder_tok_234>",
242
+ "<placeholder_tok_235>",
243
+ "<placeholder_tok_236>",
244
+ "<placeholder_tok_237>",
245
+ "<placeholder_tok_238>",
246
+ "<placeholder_tok_239>",
247
+ "<placeholder_tok_240>",
248
+ "<placeholder_tok_241>",
249
+ "<placeholder_tok_242>",
250
+ "<placeholder_tok_243>",
251
+ "<placeholder_tok_244>",
252
+ "<placeholder_tok_245>",
253
+ "<placeholder_tok_246>",
254
+ "<placeholder_tok_247>",
255
+ "<placeholder_tok_248>",
256
+ "<placeholder_tok_249>",
257
+ "<placeholder_tok_250>",
258
+ "<placeholder_tok_251>",
259
+ "<placeholder_tok_252>",
260
+ "<placeholder_tok_253>",
261
+ "<placeholder_tok_254>",
262
+ "<placeholder_tok_255>"
263
+ ],
264
+ "bos_token": "<s>",
265
+ "eos_token": "</s>",
266
+ "pad_token": "<pad>",
267
+ "unk_token": "<unk>"
268
+ }
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:08d0c8316539a853f2fe6e14f51f0df583011dfb078fa08c8b6dc5c15a19a7e6
3
+ size 4719922
tokenizer_config.json ADDED
@@ -0,0 +1,292 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "num_threads": 224,
3
+ "split_by_whitespace": true,
4
+ "model_type": "unigram",
5
+ "vocab_size": 250680,
6
+ "character_coverage": 0.9999,
7
+ "byte_fallback": true,
8
+ "split_by_number": true,
9
+ "split_digits": true,
10
+ "normalization_rule_name": "nfkc",
11
+ "max_sentence_length": 4096,
12
+ "shuffle_input_sentence": true,
13
+ "input_sentence_size": 0,
14
+ "train_extremely_large_corpus": true,
15
+ "allow_whitespace_only_pieces": true,
16
+ "required_chars": "",
17
+ "remove_extra_whitespaces": false,
18
+ "user_defined_symbols": [
19
+ "<s>",
20
+ "</s>",
21
+ "<pad>",
22
+ "<eod>",
23
+ "<placeholder_tok_0>",
24
+ "<placeholder_tok_1>",
25
+ "<placeholder_tok_2>",
26
+ "<placeholder_tok_3>",
27
+ "<placeholder_tok_4>",
28
+ "<placeholder_tok_5>",
29
+ "<placeholder_tok_6>",
30
+ "<placeholder_tok_7>",
31
+ "<placeholder_tok_8>",
32
+ "<placeholder_tok_9>",
33
+ "<placeholder_tok_10>",
34
+ "<placeholder_tok_11>",
35
+ "<placeholder_tok_12>",
36
+ "<placeholder_tok_13>",
37
+ "<placeholder_tok_14>",
38
+ "<placeholder_tok_15>",
39
+ "<placeholder_tok_16>",
40
+ "<placeholder_tok_17>",
41
+ "<placeholder_tok_18>",
42
+ "<placeholder_tok_19>",
43
+ "<placeholder_tok_20>",
44
+ "<placeholder_tok_21>",
45
+ "<placeholder_tok_22>",
46
+ "<placeholder_tok_23>",
47
+ "<placeholder_tok_24>",
48
+ "<placeholder_tok_25>",
49
+ "<placeholder_tok_26>",
50
+ "<placeholder_tok_27>",
51
+ "<placeholder_tok_28>",
52
+ "<placeholder_tok_29>",
53
+ "<placeholder_tok_30>",
54
+ "<placeholder_tok_31>",
55
+ "<placeholder_tok_32>",
56
+ "<placeholder_tok_33>",
57
+ "<placeholder_tok_34>",
58
+ "<placeholder_tok_35>",
59
+ "<placeholder_tok_36>",
60
+ "<placeholder_tok_37>",
61
+ "<placeholder_tok_38>",
62
+ "<placeholder_tok_39>",
63
+ "<placeholder_tok_40>",
64
+ "<placeholder_tok_41>",
65
+ "<placeholder_tok_42>",
66
+ "<placeholder_tok_43>",
67
+ "<placeholder_tok_44>",
68
+ "<placeholder_tok_45>",
69
+ "<placeholder_tok_46>",
70
+ "<placeholder_tok_47>",
71
+ "<placeholder_tok_48>",
72
+ "<placeholder_tok_49>",
73
+ "<placeholder_tok_50>",
74
+ "<placeholder_tok_51>",
75
+ "<placeholder_tok_52>",
76
+ "<placeholder_tok_53>",
77
+ "<placeholder_tok_54>",
78
+ "<placeholder_tok_55>",
79
+ "<placeholder_tok_56>",
80
+ "<placeholder_tok_57>",
81
+ "<placeholder_tok_58>",
82
+ "<placeholder_tok_59>",
83
+ "<placeholder_tok_60>",
84
+ "<placeholder_tok_61>",
85
+ "<placeholder_tok_62>",
86
+ "<placeholder_tok_63>",
87
+ "<placeholder_tok_64>",
88
+ "<placeholder_tok_65>",
89
+ "<placeholder_tok_66>",
90
+ "<placeholder_tok_67>",
91
+ "<placeholder_tok_68>",
92
+ "<placeholder_tok_69>",
93
+ "<placeholder_tok_70>",
94
+ "<placeholder_tok_71>",
95
+ "<placeholder_tok_72>",
96
+ "<placeholder_tok_73>",
97
+ "<placeholder_tok_74>",
98
+ "<placeholder_tok_75>",
99
+ "<placeholder_tok_76>",
100
+ "<placeholder_tok_77>",
101
+ "<placeholder_tok_78>",
102
+ "<placeholder_tok_79>",
103
+ "<placeholder_tok_80>",
104
+ "<placeholder_tok_81>",
105
+ "<placeholder_tok_82>",
106
+ "<placeholder_tok_83>",
107
+ "<placeholder_tok_84>",
108
+ "<placeholder_tok_85>",
109
+ "<placeholder_tok_86>",
110
+ "<placeholder_tok_87>",
111
+ "<placeholder_tok_88>",
112
+ "<placeholder_tok_89>",
113
+ "<placeholder_tok_90>",
114
+ "<placeholder_tok_91>",
115
+ "<placeholder_tok_92>",
116
+ "<placeholder_tok_93>",
117
+ "<placeholder_tok_94>",
118
+ "<placeholder_tok_95>",
119
+ "<placeholder_tok_96>",
120
+ "<placeholder_tok_97>",
121
+ "<placeholder_tok_98>",
122
+ "<placeholder_tok_99>",
123
+ "<placeholder_tok_100>",
124
+ "<placeholder_tok_101>",
125
+ "<placeholder_tok_102>",
126
+ "<placeholder_tok_103>",
127
+ "<placeholder_tok_104>",
128
+ "<placeholder_tok_105>",
129
+ "<placeholder_tok_106>",
130
+ "<placeholder_tok_107>",
131
+ "<placeholder_tok_108>",
132
+ "<placeholder_tok_109>",
133
+ "<placeholder_tok_110>",
134
+ "<placeholder_tok_111>",
135
+ "<placeholder_tok_112>",
136
+ "<placeholder_tok_113>",
137
+ "<placeholder_tok_114>",
138
+ "<placeholder_tok_115>",
139
+ "<placeholder_tok_116>",
140
+ "<placeholder_tok_117>",
141
+ "<placeholder_tok_118>",
142
+ "<placeholder_tok_119>",
143
+ "<placeholder_tok_120>",
144
+ "<placeholder_tok_121>",
145
+ "<placeholder_tok_122>",
146
+ "<placeholder_tok_123>",
147
+ "<placeholder_tok_124>",
148
+ "<placeholder_tok_125>",
149
+ "<placeholder_tok_126>",
150
+ "<placeholder_tok_127>",
151
+ "<placeholder_tok_128>",
152
+ "<placeholder_tok_129>",
153
+ "<placeholder_tok_130>",
154
+ "<placeholder_tok_131>",
155
+ "<placeholder_tok_132>",
156
+ "<placeholder_tok_133>",
157
+ "<placeholder_tok_134>",
158
+ "<placeholder_tok_135>",
159
+ "<placeholder_tok_136>",
160
+ "<placeholder_tok_137>",
161
+ "<placeholder_tok_138>",
162
+ "<placeholder_tok_139>",
163
+ "<placeholder_tok_140>",
164
+ "<placeholder_tok_141>",
165
+ "<placeholder_tok_142>",
166
+ "<placeholder_tok_143>",
167
+ "<placeholder_tok_144>",
168
+ "<placeholder_tok_145>",
169
+ "<placeholder_tok_146>",
170
+ "<placeholder_tok_147>",
171
+ "<placeholder_tok_148>",
172
+ "<placeholder_tok_149>",
173
+ "<placeholder_tok_150>",
174
+ "<placeholder_tok_151>",
175
+ "<placeholder_tok_152>",
176
+ "<placeholder_tok_153>",
177
+ "<placeholder_tok_154>",
178
+ "<placeholder_tok_155>",
179
+ "<placeholder_tok_156>",
180
+ "<placeholder_tok_157>",
181
+ "<placeholder_tok_158>",
182
+ "<placeholder_tok_159>",
183
+ "<placeholder_tok_160>",
184
+ "<placeholder_tok_161>",
185
+ "<placeholder_tok_162>",
186
+ "<placeholder_tok_163>",
187
+ "<placeholder_tok_164>",
188
+ "<placeholder_tok_165>",
189
+ "<placeholder_tok_166>",
190
+ "<placeholder_tok_167>",
191
+ "<placeholder_tok_168>",
192
+ "<placeholder_tok_169>",
193
+ "<placeholder_tok_170>",
194
+ "<placeholder_tok_171>",
195
+ "<placeholder_tok_172>",
196
+ "<placeholder_tok_173>",
197
+ "<placeholder_tok_174>",
198
+ "<placeholder_tok_175>",
199
+ "<placeholder_tok_176>",
200
+ "<placeholder_tok_177>",
201
+ "<placeholder_tok_178>",
202
+ "<placeholder_tok_179>",
203
+ "<placeholder_tok_180>",
204
+ "<placeholder_tok_181>",
205
+ "<placeholder_tok_182>",
206
+ "<placeholder_tok_183>",
207
+ "<placeholder_tok_184>",
208
+ "<placeholder_tok_185>",
209
+ "<placeholder_tok_186>",
210
+ "<placeholder_tok_187>",
211
+ "<placeholder_tok_188>",
212
+ "<placeholder_tok_189>",
213
+ "<placeholder_tok_190>",
214
+ "<placeholder_tok_191>",
215
+ "<placeholder_tok_192>",
216
+ "<placeholder_tok_193>",
217
+ "<placeholder_tok_194>",
218
+ "<placeholder_tok_195>",
219
+ "<placeholder_tok_196>",
220
+ "<placeholder_tok_197>",
221
+ "<placeholder_tok_198>",
222
+ "<placeholder_tok_199>",
223
+ "<placeholder_tok_200>",
224
+ "<placeholder_tok_201>",
225
+ "<placeholder_tok_202>",
226
+ "<placeholder_tok_203>",
227
+ "<placeholder_tok_204>",
228
+ "<placeholder_tok_205>",
229
+ "<placeholder_tok_206>",
230
+ "<placeholder_tok_207>",
231
+ "<placeholder_tok_208>",
232
+ "<placeholder_tok_209>",
233
+ "<placeholder_tok_210>",
234
+ "<placeholder_tok_211>",
235
+ "<placeholder_tok_212>",
236
+ "<placeholder_tok_213>",
237
+ "<placeholder_tok_214>",
238
+ "<placeholder_tok_215>",
239
+ "<placeholder_tok_216>",
240
+ "<placeholder_tok_217>",
241
+ "<placeholder_tok_218>",
242
+ "<placeholder_tok_219>",
243
+ "<placeholder_tok_220>",
244
+ "<placeholder_tok_221>",
245
+ "<placeholder_tok_222>",
246
+ "<placeholder_tok_223>",
247
+ "<placeholder_tok_224>",
248
+ "<placeholder_tok_225>",
249
+ "<placeholder_tok_226>",
250
+ "<placeholder_tok_227>",
251
+ "<placeholder_tok_228>",
252
+ "<placeholder_tok_229>",
253
+ "<placeholder_tok_230>",
254
+ "<placeholder_tok_231>",
255
+ "<placeholder_tok_232>",
256
+ "<placeholder_tok_233>",
257
+ "<placeholder_tok_234>",
258
+ "<placeholder_tok_235>",
259
+ "<placeholder_tok_236>",
260
+ "<placeholder_tok_237>",
261
+ "<placeholder_tok_238>",
262
+ "<placeholder_tok_239>",
263
+ "<placeholder_tok_240>",
264
+ "<placeholder_tok_241>",
265
+ "<placeholder_tok_242>",
266
+ "<placeholder_tok_243>",
267
+ "<placeholder_tok_244>",
268
+ "<placeholder_tok_245>",
269
+ "<placeholder_tok_246>",
270
+ "<placeholder_tok_247>",
271
+ "<placeholder_tok_248>",
272
+ "<placeholder_tok_249>",
273
+ "<placeholder_tok_250>",
274
+ "<placeholder_tok_251>",
275
+ "<placeholder_tok_252>",
276
+ "<placeholder_tok_253>",
277
+ "<placeholder_tok_254>",
278
+ "<placeholder_tok_255>"
279
+ ],
280
+ "datasets_dir": "/home/fhgiais/gptx_ablations/bias_analysis/data/tokenizer/temp/",
281
+ "save_dir": "/home/fhgiais/gptx_ablations/bias_analysis/tokenizer/24",
282
+ "text_key": "text",
283
+ "cache_dir": "/home/fhgiais/gptx_ablations/bias_analysis/tokenizer/24/cache",
284
+ "library": "sentencepiece",
285
+ "auto_map": {
286
+ "AutoTokenizer": [
287
+ "gptx_tokenizer.SPTokenizer",
288
+ null
289
+ ]
290
+ },
291
+ "tokenizer_class": "SPTokenizer"
292
+ }