Hugging Face's logo

Changlong1
/
ttLlama-7b

Text Generation
Transformers
Safetensors
llama
text-generation-inference
Inference Endpoints
Model card Files Community
Changlong1 commited on
Commit
310c653
·
1 Parent(s): 6069783

Upload tokenizer

Browse files
Files changed (3) hide show
  1. special_tokens_map.json +9 -4
  2. tokenizer.json +0 -0
  3. tokenizer_config.json +49 -11
special_tokens_map.json CHANGED
@@ -1,23 +1,28 @@
1
  {
 
 
 
 
 
 
2
  "bos_token": {
3
  "content": "<s>",
4
  "lstrip": false,
5
- "normalized": false,
6
  "rstrip": false,
7
  "single_word": false
8
  },
9
  "eos_token": {
10
  "content": "</s>",
11
  "lstrip": false,
12
- "normalized": false,
13
  "rstrip": false,
14
  "single_word": false
15
  },
16
- "pad_token": "</s>",
17
  "unk_token": {
18
  "content": "<unk>",
19
  "lstrip": false,
20
- "normalized": false,
21
  "rstrip": false,
22
  "single_word": false
23
  }
 
1
  {
2
+ "additional_special_tokens": [
3
+ "▁<PRE>",
4
+ "▁<MID>",
5
+ "▁<SUF>",
6
+ "▁<EOT>"
7
+ ],
8
  "bos_token": {
9
  "content": "<s>",
10
  "lstrip": false,
11
+ "normalized": true,
12
  "rstrip": false,
13
  "single_word": false
14
  },
15
  "eos_token": {
16
  "content": "</s>",
17
  "lstrip": false,
18
+ "normalized": true,
19
  "rstrip": false,
20
  "single_word": false
21
  },
 
22
  "unk_token": {
23
  "content": "<unk>",
24
  "lstrip": false,
25
+ "normalized": true,
26
  "rstrip": false,
27
  "single_word": false
28
  }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json CHANGED
@@ -1,11 +1,9 @@
1
  {
2
- "add_bos_token": true,
3
- "add_eos_token": false,
4
  "added_tokens_decoder": {
5
  "0": {
6
  "content": "<unk>",
7
  "lstrip": false,
8
- "normalized": false,
9
  "rstrip": false,
10
  "single_word": false,
11
  "special": true
@@ -13,7 +11,7 @@
13
  "1": {
14
  "content": "<s>",
15
  "lstrip": false,
16
- "normalized": false,
17
  "rstrip": false,
18
  "single_word": false,
19
  "special": true
@@ -21,23 +19,63 @@
21
  "2": {
22
  "content": "</s>",
23
  "lstrip": false,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
24
  "normalized": false,
25
  "rstrip": false,
26
  "single_word": false,
27
  "special": true
28
  }
29
  },
 
 
 
 
 
 
30
  "bos_token": "<s>",
31
  "clean_up_tokenization_spaces": false,
32
  "eos_token": "</s>",
33
- "legacy": true,
 
 
 
34
  "model_max_length": 1000000000000000019884624838656,
35
- "pad_token": "</s>",
 
36
  "sp_model_kwargs": {},
37
- "spaces_between_special_tokens": false,
38
- "tokenizer_class": "LlamaTokenizer",
39
- "trust_remote_code": false,
40
  "unk_token": "<unk>",
41
- "use_default_system_prompt": false,
42
- "use_fast": true
43
  }
 
1
  {
 
 
2
  "added_tokens_decoder": {
3
  "0": {
4
  "content": "<unk>",
5
  "lstrip": false,
6
+ "normalized": true,
7
  "rstrip": false,
8
  "single_word": false,
9
  "special": true
 
11
  "1": {
12
  "content": "<s>",
13
  "lstrip": false,
14
+ "normalized": true,
15
  "rstrip": false,
16
  "single_word": false,
17
  "special": true
 
19
  "2": {
20
  "content": "</s>",
21
  "lstrip": false,
22
+ "normalized": true,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "32007": {
28
+ "content": "▁<PRE>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "32008": {
36
+ "content": "▁<SUF>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ },
43
+ "32009": {
44
+ "content": "▁<MID>",
45
+ "lstrip": false,
46
+ "normalized": false,
47
+ "rstrip": false,
48
+ "single_word": false,
49
+ "special": true
50
+ },
51
+ "32010": {
52
+ "content": "▁<EOT>",
53
+ "lstrip": false,
54
  "normalized": false,
55
  "rstrip": false,
56
  "single_word": false,
57
  "special": true
58
  }
59
  },
60
+ "additional_special_tokens": [
61
+ "▁<PRE>",
62
+ "▁<MID>",
63
+ "▁<SUF>",
64
+ "▁<EOT>"
65
+ ],
66
  "bos_token": "<s>",
67
  "clean_up_tokenization_spaces": false,
68
  "eos_token": "</s>",
69
+ "eot_token": "▁<EOT>",
70
+ "fill_token": "<FILL_ME>",
71
+ "legacy": null,
72
+ "middle_token": "▁<MID>",
73
  "model_max_length": 1000000000000000019884624838656,
74
+ "pad_token": null,
75
+ "prefix_token": "▁<PRE>",
76
  "sp_model_kwargs": {},
77
+ "suffix_token": "▁<SUF>",
78
+ "tokenizer_class": "CodeLlamaTokenizer",
 
79
  "unk_token": "<unk>",
80
+ "use_default_system_prompt": false
 
81
  }