Files changed (3) hide show
  1. special_tokens_map.json +21 -3
  2. tokenizer.json +7 -16
  3. tokenizer_config.json +2 -1
special_tokens_map.json CHANGED
@@ -1,5 +1,23 @@
1
  {
2
- "bos_token": "<s>",
3
- "eos_token": "</s>",
4
- "unk_token": "<unk>"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5
  }
 
1
  {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "unk_token": {
17
+ "content": "<unk>",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ }
23
  }
tokenizer.json CHANGED
@@ -31,23 +31,13 @@
31
  "special": true
32
  }
33
  ],
34
- "normalizer": {
35
- "type": "Sequence",
36
- "normalizers": [
37
- {
38
- "type": "Prepend",
39
- "prepend": "▁"
40
- },
41
- {
42
- "type": "Replace",
43
- "pattern": {
44
- "String": " "
45
- },
46
- "content": "▁"
47
- }
48
- ]
49
  },
50
- "pre_tokenizer": null,
51
  "post_processor": {
52
  "type": "TemplateProcessing",
53
  "single": [
@@ -134,6 +124,7 @@
134
  "end_of_word_suffix": null,
135
  "fuse_unk": true,
136
  "byte_fallback": true,
 
137
  "vocab": {
138
  "<unk>": 0,
139
  "<s>": 1,
 
31
  "special": true
32
  }
33
  ],
34
+ "normalizer": null,
35
+ "pre_tokenizer": {
36
+ "type": "Metaspace",
37
+ "replacement": "▁",
38
+ "prepend_scheme": "first",
39
+ "split": false
 
 
 
 
 
 
 
 
 
40
  },
 
41
  "post_processor": {
42
  "type": "TemplateProcessing",
43
  "single": [
 
124
  "end_of_word_suffix": null,
125
  "fuse_unk": true,
126
  "byte_fallback": true,
127
+ "ignore_merges": false,
128
  "vocab": {
129
  "<unk>": 0,
130
  "<s>": 1,
tokenizer_config.json CHANGED
@@ -1,6 +1,7 @@
1
  {
2
  "add_bos_token": true,
3
  "add_eos_token": false,
 
4
  "added_tokens_decoder": {
5
  "0": {
6
  "content": "<unk>",
@@ -31,7 +32,7 @@
31
  "bos_token": "<s>",
32
  "clean_up_tokenization_spaces": false,
33
  "eos_token": "</s>",
34
- "legacy": true,
35
  "model_max_length": 1000000000000000019884624838656,
36
  "pad_token": null,
37
  "sp_model_kwargs": {},
 
1
  {
2
  "add_bos_token": true,
3
  "add_eos_token": false,
4
+ "add_prefix_space": null,
5
  "added_tokens_decoder": {
6
  "0": {
7
  "content": "<unk>",
 
32
  "bos_token": "<s>",
33
  "clean_up_tokenization_spaces": false,
34
  "eos_token": "</s>",
35
+ "legacy": false,
36
  "model_max_length": 1000000000000000019884624838656,
37
  "pad_token": null,
38
  "sp_model_kwargs": {},