crossdelenna commited on
Commit
13873ea
1 Parent(s): ce73eba

add tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +1 -1
vocab.json CHANGED
@@ -1 +1 @@
1
- {"r": 0, "g": 1, "n": 2, "z": 3, "p": 4, "t": 5, "=": 6, "l": 7, "b": 8, "v": 9, "k": 10, "u": 11, "-": 12, "e": 13, "o": 14, "j": 15, "x": 16, "q": 17, ",": 18, "m": 19, "d": 20, "f": 21, "s": 22, "'": 23, "a": 25, "%": 26, "c": 27, "i": 28, "w": 29, "y": 30, "h": 31, "|": 24, "[UNK]": 32, "[PAD]": 33}
 
1
+ {"b": 0, "o": 1, "q": 2, "l": 3, "w": 4, "y": 5, "x": 6, "e": 7, "a": 8, "z": 9, "-": 10, "f": 11, "k": 12, "u": 13, "c": 14, "p": 15, "'": 16, "n": 17, "t": 18, "g": 19, "d": 20, "m": 22, ",": 23, "r": 24, "v": 25, "s": 26, "i": 27, "j": 28, "h": 29, "|": 21, "[UNK]": 30, "[PAD]": 31}