TiktokenTokenizer: upload tokenizer_config.json

tokenizer_config.json

@@ -1,19 +1,26 @@
 {
-  "add_prefix_space": false,
-  "backend": "tokenizers",
-  "bos_token": "éĤĢ",
-  "clean_up_tokenization_spaces": false,
-  "eos_token": "éĤĢ",
-  "errors": "replace",
-  "extra_special_tokens": [
-    "<|im_start|>",
-    "<|im_end|>"
-  ],
-  "is_local": false,
-  "local_files_only": false,
-  "model_max_length": 32768,
-  "pad_token": "éĤĢ",
-  "split_special_tokens": false,
-  "tokenizer_class": "Qwen2Tokenizer",
-  "unk_token": null
-}
+  "added_tokens_decoder": {
+    "100257": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "backend": "custom",
+  "bos_token": "<|endoftext|>",
+  "encoding_name": "cl100k_base",
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "TiktokenTokenizer",
+  "unk_token": "<|endoftext|>",
+  "auto_map": {
+    "AutoTokenizer": [
+      null,
+      "tokenization_parchment.TiktokenTokenizer"
+    ]
+  }
+}