achalagarwal commited on
Commit
b46bda9
1 Parent(s): 7ee7b6b

Update tokenizer_config.json

Browse files

Following https://huggingface.co/codellama/CodeLlama-34b-hf/discussions/9 to support usage with ctranslate2
cc:

@ctranslate2-4you

Files changed (1) hide show
  1. tokenizer_config.json +7 -0
tokenizer_config.json CHANGED
@@ -1,6 +1,7 @@
1
  {
2
  "add_bos_token": true,
3
  "add_eos_token": false,
 
4
  "bos_token": {
5
  "__type": "AddedToken",
6
  "content": "<s>",
@@ -9,6 +10,8 @@
9
  "rstrip": false,
10
  "single_word": false
11
  },
 
 
12
  "clean_up_tokenization_spaces": false,
13
  "eos_token": {
14
  "__type": "AddedToken",
@@ -19,9 +22,13 @@
19
  "single_word": false
20
  },
21
  "legacy": null,
 
22
  "model_max_length": 1000000000000000019884624838656,
23
  "pad_token": null,
 
24
  "sp_model_kwargs": {},
 
 
25
  "tokenizer_class": "CodeLlamaTokenizer",
26
  "unk_token": {
27
  "__type": "AddedToken",
 
1
  {
2
  "add_bos_token": true,
3
  "add_eos_token": false,
4
+ "additional_special_tokens": [],
5
  "bos_token": {
6
  "__type": "AddedToken",
7
  "content": "<s>",
 
10
  "rstrip": false,
11
  "single_word": false
12
  },
13
+ "eot_token": null,
14
+ "fill_token": "<FILL_ME>",
15
  "clean_up_tokenization_spaces": false,
16
  "eos_token": {
17
  "__type": "AddedToken",
 
22
  "single_word": false
23
  },
24
  "legacy": null,
25
+ "middle_token": null,
26
  "model_max_length": 1000000000000000019884624838656,
27
  "pad_token": null,
28
+ "prefix_token": null,
29
  "sp_model_kwargs": {},
30
+ "suffix_first": false,
31
+ "suffix_token": null,
32
  "tokenizer_class": "CodeLlamaTokenizer",
33
  "unk_token": {
34
  "__type": "AddedToken",