devops-thiago commited on
Commit
0a02d89
·
verified ·
1 Parent(s): 75f972f

Add ClassOne delimiter config to published tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +13 -2
tokenizer_config.json CHANGED
@@ -14,10 +14,21 @@
14
  "etd_token": "<tool|>",
15
  "etr_token": "<tool_response|>",
16
  "extra_special_tokens": [
17
- "<|video|>"
 
 
 
 
 
 
 
 
 
 
 
18
  ],
19
  "image_token": "<|image|>",
20
- "is_local": false,
21
  "local_files_only": false,
22
  "mask_token": "<mask>",
23
  "model_max_length": 1000000000000000019884624838656,
 
14
  "etd_token": "<tool|>",
15
  "etr_token": "<tool_response|>",
16
  "extra_special_tokens": [
17
+ "<|state_start|>",
18
+ "<|state_end|>",
19
+ "<|noul_start|>",
20
+ "<|noul_end|>",
21
+ "<|choice_start|>",
22
+ "<|choice_end|>",
23
+ "<|opt_start|>",
24
+ "<|opt_end|>",
25
+ "<|score_start|>",
26
+ "<|score_end|>",
27
+ "<|level_start|>",
28
+ "<|level_end|>"
29
  ],
30
  "image_token": "<|image|>",
31
+ "is_local": true,
32
  "local_files_only": false,
33
  "mask_token": "<mask>",
34
  "model_max_length": 1000000000000000019884624838656,