Salmamoori commited on
Commit
4556eee
1 Parent(s): 8f3d761

Upload processor

Browse files
Files changed (2) hide show
  1. tokenizer.json +1 -8
  2. tokenizer_config.json +1 -2
tokenizer.json CHANGED
@@ -1,14 +1,7 @@
1
  {
2
  "version": "1.0",
3
  "truncation": null,
4
- "padding": {
5
- "strategy": "BatchLongest",
6
- "direction": "Right",
7
- "pad_to_multiple_of": null,
8
- "pad_id": 32001,
9
- "pad_type_id": 0,
10
- "pad_token": "<pad>"
11
- },
12
  "added_tokens": [
13
  {
14
  "id": 0,
 
1
  {
2
  "version": "1.0",
3
  "truncation": null,
4
+ "padding": null,
 
 
 
 
 
 
 
5
  "added_tokens": [
6
  {
7
  "id": 0,
tokenizer_config.json CHANGED
@@ -45,13 +45,12 @@
45
  }
46
  },
47
  "bos_token": "<s>",
48
- "chat_template": "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. {% for message in messages %}{% if message['role'] == 'user' %}USER: {% else %}ASSISTANT: {% endif %}{% for item in message['content'] %}{% if item['type'] == 'text' %}{{ item['text'] }}{% elif item['type'] == 'image' %}<image>{% endif %}{% endfor %}{% if message['role'] == 'user' %} {% else %}{{eos_token}}{% endif %}{% endfor %}",
49
  "clean_up_tokenization_spaces": false,
50
  "eos_token": "</s>",
51
  "legacy": false,
52
  "model_max_length": 1000000000000000019884624838656,
53
  "pad_token": "<pad>",
54
- "padding_side": "right",
55
  "processor_class": "LlavaProcessor",
56
  "sp_model_kwargs": {},
57
  "tokenizer_class": "LlamaTokenizer",
 
45
  }
46
  },
47
  "bos_token": "<s>",
 
48
  "clean_up_tokenization_spaces": false,
49
  "eos_token": "</s>",
50
  "legacy": false,
51
  "model_max_length": 1000000000000000019884624838656,
52
  "pad_token": "<pad>",
53
+ "padding_side": "left",
54
  "processor_class": "LlavaProcessor",
55
  "sp_model_kwargs": {},
56
  "tokenizer_class": "LlamaTokenizer",