kgrabko commited on
Commit
703ce8f
·
verified ·
1 Parent(s): fea9e79

updated upload of JiRackTokenizer with improvement for Robotics and Routing tags

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +2 -1
tokenizer_config.json CHANGED
@@ -114,12 +114,13 @@
114
  "__TRAVEL__",
115
  "__HOBBY__"
116
  ],
117
- "is_local": false,
118
  "local_files_only": false,
119
  "model_input_names": [
120
  "input_ids",
121
  "attention_mask"
122
  ],
123
  "model_max_length": 1000000000000000019884624838656,
 
124
  "tokenizer_class": "TokenizersBackend"
125
  }
 
114
  "__TRAVEL__",
115
  "__HOBBY__"
116
  ],
117
+ "is_local": true,
118
  "local_files_only": false,
119
  "model_input_names": [
120
  "input_ids",
121
  "attention_mask"
122
  ],
123
  "model_max_length": 1000000000000000019884624838656,
124
+ "pad_token": "<|end_of_text|>",
125
  "tokenizer_class": "TokenizersBackend"
126
  }