Hugging Face's logo

allenai
/
open-instruct-human-mix-65b

Text Generation
Transformers
PyTorch
English
llama
text-generation-inference
Inference Endpoints
Model card Files Community
hamishivi commited on
Commit
5874dd3
•
1 Parent(s): d01234a

tokenizer fix

Browse files
Files changed (2) hide show
  1. tokenizer.json +0 -9
  2. tokenizer_config.json +4 -23
tokenizer.json CHANGED
@@ -100,15 +100,6 @@
100
  }
101
  ],
102
  "special_tokens": {
103
- "</s>": {
104
- "id": "</s>",
105
- "ids": [
106
- 2
107
- ],
108
- "tokens": [
109
- "</s>"
110
- ]
111
- },
112
  "<s>": {
113
  "id": "<s>",
114
  "ids": [
 
100
  }
101
  ],
102
  "special_tokens": {
 
 
 
 
 
 
 
 
 
103
  "<s>": {
104
  "id": "<s>",
105
  "ids": [
tokenizer_config.json CHANGED
@@ -1,33 +1,14 @@
1
  {
2
- "add_bos_token": true,
3
- "add_eos_token": false,
4
- "bos_token": {
5
- "__type": "AddedToken",
6
- "content": "",
7
- "lstrip": false,
8
- "normalized": true,
9
- "rstrip": false,
10
- "single_word": false
11
- },
12
  "clean_up_tokenization_spaces": false,
13
- "eos_token": {
 
14
  "__type": "AddedToken",
15
- "content": "",
16
  "lstrip": false,
17
  "normalized": true,
18
  "rstrip": false,
19
  "single_word": false
20
  },
21
- "model_max_length": 1000000000000000019884624838656,
22
- "pad_token": null,
23
  "sp_model_kwargs": {},
24
- "tokenizer_class": "LlamaTokenizer",
25
- "unk_token": {
26
- "__type": "AddedToken",
27
- "content": "",
28
- "lstrip": false,
29
- "normalized": true,
30
- "rstrip": false,
31
- "single_word": false
32
- }
33
  }
 
1
  {
 
 
 
 
 
 
 
 
 
 
2
  "clean_up_tokenization_spaces": false,
3
+ "model_max_length": 1000000000000000019884624838656,
4
+ "pad_token": {
5
  "__type": "AddedToken",
6
+ "content": "<pad>",
7
  "lstrip": false,
8
  "normalized": true,
9
  "rstrip": false,
10
  "single_word": false
11
  },
 
 
12
  "sp_model_kwargs": {},
13
+ "tokenizer_class": "LlamaTokenizer"
 
 
 
 
 
 
 
 
14
  }