Upload tokenizer_config.json

tokenizer_config.json

@@ -1,16 +1,14 @@
 {
-  "add_prefix_space": true,
-  "backend": "tokenizers",
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_llama.LlamaTokenizer",
+      null
+    ]
+  },
   "bos_token": "<s>",
-  "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
-  "is_local": true,
-  "legacy": false,
-  "model_max_length": 512,
   "pad_token": "</s>",
-  "sp_model_kwargs": {},
-  "spaces_between_special_tokens": false,
-  "tokenizer_class": "TokenizersBackend",
+  "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",
-  "use_default_system_prompt": false
-}
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{{ messages[0]['content'] }}{% endif %}{% for message in messages %}{% if message['role'] == 'user' %}### Task: {{ message['content'] }}\n\n### Solution:\n{% elif message['role'] == 'assistant' %}{{ message['content'] }}{% endif %}{% endfor %}"
+}