ArthurZ HF staff altryne commited on
Commit
8050949
1 Parent(s): 18af65e

change hebrew code from iw (deprecated) to he (#8)

Browse files

- change hebrew code from iw (deprecated) to he (106e8e1eaa5586609eeea0529217813730d8611c)


Co-authored-by: Altryne <altryne@users.noreply.huggingface.co>

README.md CHANGED
@@ -20,7 +20,7 @@ language:
20
  - hi
21
  - fi
22
  - vi
23
- - iw
24
  - uk
25
  - el
26
  - ms
 
20
  - hi
21
  - fi
22
  - vi
23
+ - he
24
  - uk
25
  - el
26
  - ms
added_tokens.json CHANGED
@@ -29,6 +29,7 @@
29
  "<|gu|>": 50333,
30
  "<|haw|>": 50352,
31
  "<|ha|>": 50354,
 
32
  "<|hi|>": 50276,
33
  "<|hr|>": 50291,
34
  "<|ht|>": 50339,
@@ -37,7 +38,6 @@
37
  "<|id|>": 50275,
38
  "<|is|>": 50311,
39
  "<|it|>": 50274,
40
- "<|iw|>": 50279,
41
  "<|ja|>": 50266,
42
  "<|jw|>": 50356,
43
  "<|ka|>": 50329,
 
29
  "<|gu|>": 50333,
30
  "<|haw|>": 50352,
31
  "<|ha|>": 50354,
32
+ "<|he|>": 50279,
33
  "<|hi|>": 50276,
34
  "<|hr|>": 50291,
35
  "<|ht|>": 50339,
 
38
  "<|id|>": 50275,
39
  "<|is|>": 50311,
40
  "<|it|>": 50274,
 
41
  "<|ja|>": 50266,
42
  "<|jw|>": 50356,
43
  "<|ka|>": 50329,
generation_config.json CHANGED
@@ -48,6 +48,7 @@
48
  "<|gu|>": 50333,
49
  "<|haw|>": 50352,
50
  "<|ha|>": 50354,
 
51
  "<|hi|>": 50276,
52
  "<|hr|>": 50291,
53
  "<|ht|>": 50339,
@@ -56,7 +57,6 @@
56
  "<|id|>": 50275,
57
  "<|is|>": 50311,
58
  "<|it|>": 50274,
59
- "<|iw|>": 50279,
60
  "<|ja|>": 50266,
61
  "<|jw|>": 50356,
62
  "<|ka|>": 50329,
 
48
  "<|gu|>": 50333,
49
  "<|haw|>": 50352,
50
  "<|ha|>": 50354,
51
+ "<|he|>": 50279,
52
  "<|hi|>": 50276,
53
  "<|hr|>": 50291,
54
  "<|ht|>": 50339,
 
57
  "<|id|>": 50275,
58
  "<|is|>": 50311,
59
  "<|it|>": 50274,
 
60
  "<|ja|>": 50266,
61
  "<|jw|>": 50356,
62
  "<|ka|>": 50329,
special_tokens_map.json CHANGED
@@ -22,7 +22,7 @@
22
  "<|hi|>",
23
  "<|fi|>",
24
  "<|vi|>",
25
- "<|iw|>",
26
  "<|uk|>",
27
  "<|el|>",
28
  "<|ms|>",
 
22
  "<|hi|>",
23
  "<|fi|>",
24
  "<|vi|>",
25
+ "<|he|>",
26
  "<|uk|>",
27
  "<|el|>",
28
  "<|ms|>",
tokenizer.json CHANGED
@@ -203,7 +203,7 @@
203
  },
204
  {
205
  "id": 50279,
206
- "content": "<|iw|>",
207
  "single_word": false,
208
  "lstrip": false,
209
  "rstrip": false,
 
203
  },
204
  {
205
  "id": 50279,
206
+ "content": "<|he|>",
207
  "single_word": false,
208
  "lstrip": false,
209
  "rstrip": false,