Hugging Face's logo

tinisoft
/
SmolLM-135M-addition

Text Generation
Transformers
Safetensors
llama
trl
sft
conversational
text-generation-inference
Inference Endpoints
Model card Files Community
aravindhank commited on
Commit
3166dc4
·
verified ·
1 Parent(s): bdb5ebe

Upload tokenizer

Browse files
Files changed (2) hide show
  1. special_tokens_map.json +17 -32
  2. tokenizer_config.json +6 -20
special_tokens_map.json CHANGED
@@ -1,38 +1,23 @@
1
  {
2
  "additional_special_tokens": [
3
- "<|endoftext|>",
4
- "<|im_start|>",
5
- "<|im_end|>",
6
- "<repo_name>",
7
- "<reponame>",
8
- "<file_sep>",
9
- "<filename>",
10
- "<gh_stars>",
11
- "<issue_start>",
12
- "<issue_comment>",
13
- "<issue_closed>",
14
- "<jupyter_start>",
15
- "<jupyter_text>",
16
- "<jupyter_code>",
17
- "<jupyter_output>",
18
- "<jupyter_script>",
19
- "<empty_output>"
20
  ],
21
- "bos_token": {
22
- "content": "<|endoftext|>",
23
- "lstrip": false,
24
- "normalized": false,
25
- "rstrip": false,
26
- "single_word": false
27
- },
28
- "eos_token": {
29
- "content": "<|endoftext|>",
30
- "lstrip": false,
31
- "normalized": false,
32
- "rstrip": false,
33
- "single_word": false
34
- },
35
- "pad_token": "<|endoftext|>",
36
  "unk_token": {
37
  "content": "<|endoftext|>",
38
  "lstrip": false,
 
1
  {
2
  "additional_special_tokens": [
3
+ {
4
+ "content": "<|im_start|>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false
9
+ },
10
+ {
11
+ "content": "<|im_end|>",
12
+ "lstrip": false,
13
+ "normalized": false,
14
+ "rstrip": false,
15
+ "single_word": false
16
+ }
 
 
 
17
  ],
18
+ "bos_token": "<|im_start|>",
19
+ "eos_token": "<|im_end|>",
20
+ "pad_token": "<|im_end|>",
 
 
 
 
 
 
 
 
 
 
 
 
21
  "unk_token": {
22
  "content": "<|endoftext|>",
23
  "lstrip": false,
tokenizer_config.json CHANGED
@@ -139,30 +139,16 @@
139
  }
140
  },
141
  "additional_special_tokens": [
142
- "<|endoftext|>",
143
  "<|im_start|>",
144
- "<|im_end|>",
145
- "<repo_name>",
146
- "<reponame>",
147
- "<file_sep>",
148
- "<filename>",
149
- "<gh_stars>",
150
- "<issue_start>",
151
- "<issue_comment>",
152
- "<issue_closed>",
153
- "<jupyter_start>",
154
- "<jupyter_text>",
155
- "<jupyter_code>",
156
- "<jupyter_output>",
157
- "<jupyter_script>",
158
- "<empty_output>"
159
  ],
160
- "bos_token": "<|endoftext|>",
 
161
  "clean_up_tokenization_spaces": false,
162
- "eos_token": "<|endoftext|>",
163
  "extra_special_tokens": {},
164
- "model_max_length": 1000000000000000019884624838656,
165
- "pad_token": "<|endoftext|>",
166
  "tokenizer_class": "GPT2Tokenizer",
167
  "unk_token": "<|endoftext|>",
168
  "vocab_size": 49152
 
139
  }
140
  },
141
  "additional_special_tokens": [
 
142
  "<|im_start|>",
143
+ "<|im_end|>"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
144
  ],
145
+ "bos_token": "<|im_start|>",
146
+ "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
147
  "clean_up_tokenization_spaces": false,
148
+ "eos_token": "<|im_end|>",
149
  "extra_special_tokens": {},
150
+ "model_max_length": 8192,
151
+ "pad_token": "<|im_end|>",
152
  "tokenizer_class": "GPT2Tokenizer",
153
  "unk_token": "<|endoftext|>",
154
  "vocab_size": 49152