Isaak Carter Augustus commited on
Commit
9dd132d
1 Parent(s): be10e90

Upload tokenizer

Browse files
added_tokens.json ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "<|assistant|>": 32775,
3
+ "<|available_functions|>": 32770,
4
+ "<|context|>": 32783,
5
+ "<|current_states|>": 32782,
6
+ "<|endoftext|>": 32769,
7
+ "<|function_call|>": 32776,
8
+ "<|function_response|>": 32777,
9
+ "<|gökdeniz|>": 32772,
10
+ "<|home_state|>": 32781,
11
+ "<|im_end|>": 32785,
12
+ "<|im_start|>": 32784,
13
+ "<|image|>": 32778,
14
+ "<|josie|>": 32774,
15
+ "<|long_term_memory|>": 32779,
16
+ "<|short_term_memory|>": 32780,
17
+ "<|startoftext|>": 32768,
18
+ "<|system|>": 32771,
19
+ "<|user|>": 32773
20
+ }
special_tokens_map.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|available_functions|>",
4
+ "<|system|>",
5
+ "<|gökdeniz|>",
6
+ "<|user|>",
7
+ "<|josie|>",
8
+ "<|assistant|>",
9
+ "<|function_call|>",
10
+ "<|function_response|>",
11
+ "<|image|>",
12
+ "<|long_term_memory|>",
13
+ "<|short_term_memory|>",
14
+ "<|home_state|>",
15
+ "<|current_states|>",
16
+ "<|context|>",
17
+ "<|im_start|>",
18
+ "<|im_end|>"
19
+ ],
20
+ "bos_token": {
21
+ "content": "<|startoftext|>",
22
+ "lstrip": false,
23
+ "normalized": false,
24
+ "rstrip": false,
25
+ "single_word": false
26
+ },
27
+ "eos_token": {
28
+ "content": "<|endoftext|>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false
33
+ },
34
+ "pad_token": "[control_768]",
35
+ "unk_token": {
36
+ "content": "<unk>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false
41
+ }
42
+ }
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:37f00374dea48658ee8f5d0f21895b9bc55cb0103939607c8185bfd1c6ca1f89
3
+ size 587404
tokenizer_config.json ADDED
The diff for this file is too large to render. See raw diff