Upload folder using huggingface_hub

special_tokens_map.json

@@ -0,0 +1,286 @@
+{
+  "pad_token": "<|pad|>",
+  "bos_token": "<|startoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|unk|>",
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|tool_call_start|>",
+    "<|tool_call_end|>",
+    "<|tool_response_start|>",
+    "<|tool_response_end|>",
+    "<|reasoning_start|>",
+    "<|reasoning_end|>",
+    "<|fim_prefix|>",
+    "<|fim_middle|>",
+    "<|fim_suffix|>",
+    "<|fim_pad|>",
+    "<|repo_name|>",
+    "<|file_sep|>",
+    "<|lang:agu|>",
+    "<|lang:all|>",
+    "<|lang:am1*|>",
+    "<|lang:amu|>",
+    "<|lang:amu0|>",
+    "<|lang:azg|>",
+    "<|lang:azn|>",
+    "<|lang:azn0|>",
+    "<|lang:azz|>",
+    "<|lang:cac|>",
+    "<|lang:cak|>",
+    "<|lang:cco|>",
+    "<|lang:cco0|>",
+    "<|lang:chd|>",
+    "<|lang:chf|>",
+    "<|lang:chf0|>",
+    "<|lang:chj|>",
+    "<|lang:chq|>",
+    "<|lang:chz|>",
+    "<|lang:cle|>",
+    "<|lang:cnl|>",
+    "<|lang:cnt|>",
+    "<|lang:cok|>",
+    "<|lang:coz|>",
+    "<|lang:cpa|>",
+    "<|lang:crn|>",
+    "<|lang:cso|>",
+    "<|lang:ct2*|>",
+    "<|lang:cta|>",
+    "<|lang:cte|>",
+    "<|lang:ctl|>",
+    "<|lang:ctp|>",
+    "<|lang:ctp0|>",
+    "<|lang:ctu|>",
+    "<|lang:cuc|>",
+    "<|lang:cut|>",
+    "<|lang:cux|>",
+    "<|lang:de|>",
+    "<|lang:dih|>",
+    "<|lang:en-US|>",
+    "<|lang:eng|>",
+    "<|lang:es-MX|>",
+    "<|lang:esp|>",
+    "<|lang:fr|>",
+    "<|lang:hch|>",
+    "<|lang:hu2*|>",
+    "<|lang:hus|>",
+    "<|lang:huv|>",
+    "<|lang:hve|>",
+    "<|lang:hve0|>",
+    "<|lang:ixc|>",
+    "<|lang:jac|>",
+    "<|lang:jmx|>",
+    "<|lang:kak|>",
+    "<|lang:kek|>",
+    "<|lang:kic|>",
+    "<|lang:klb|>",
+    "<|lang:knj|>",
+    "<|lang:knja|>",
+    "<|lang:ku1*|>",
+    "<|lang:lac|>",
+    "<|lang:maa|>",
+    "<|lang:maj|>",
+    "<|lang:mam|>",
+    "<|lang:maq|>",
+    "<|lang:mat|>",
+    "<|lang:mau|>",
+    "<|lang:maz|>",
+    "<|lang:maz0|>",
+    "<|lang:mbz|>",
+    "<|lang:mbzl|>",
+    "<|lang:mco|>",
+    "<|lang:mco0|>",
+    "<|lang:meh|>",
+    "<|lang:mfy|>",
+    "<|lang:mhc|>",
+    "<|lang:mib|>",
+    "<|lang:mie|>",
+    "<|lang:mig|>",
+    "<|lang:mih|>",
+    "<|lang:mil|>",
+    "<|lang:mim|>",
+    "<|lang:mim0|>",
+    "<|lang:mio|>",
+    "<|lang:mir|>",
+    "<|lang:mit|>",
+    "<|lang:mix|>",
+    "<|lang:miy|>",
+    "<|lang:miz|>",
+    "<|lang:mjc|>",
+    "<|lang:mks|>",
+    "<|lang:ml2*|>",
+    "<|lang:ml8*|>",
+    "<|lang:mmc|>",
+    "<|lang:mop|>",
+    "<|lang:mpm|>",
+    "<|lang:mto|>",
+    "<|lang:mxb|>",
+    "<|lang:mxp|>",
+    "<|lang:mxq|>",
+    "<|lang:mxt|>",
+    "<|lang:mxv|>",
+    "<|lang:mxv0|>",
+    "<|lang:mxy|>",
+    "<|lang:mza|>",
+    "<|lang:mzi|>",
+    "<|lang:mzia|>",
+    "<|lang:naz|>",
+    "<|lang:nch|>",
+    "<|lang:ncj|>",
+    "<|lang:ncl|>",
+    "<|lang:ncx|>",
+    "<|lang:neq|>",
+    "<|lang:ngu|>",
+    "<|lang:nhe|>",
+    "<|lang:nhg|>",
+    "<|lang:nhi|>",
+    "<|lang:nhm|>",
+    "<|lang:nhn|>",
+    "<|lang:nhw|>",
+    "<|lang:nhx|>",
+    "<|lang:nhy|>",
+    "<|lang:nlv|>",
+    "<|lang:nlv0|>",
+    "<|lang:npl|>",
+    "<|lang:nsu|>",
+    "<|lang:ntp|>",
+    "<|lang:ocu|>",
+    "<|lang:ood|>",
+    "<|lang:opt|>",
+    "<|lang:ote|>",
+    "<|lang:otm|>",
+    "<|lang:otn|>",
+    "<|lang:otq|>",
+    "<|lang:ots|>",
+    "<|lang:ott|>",
+    "<|lang:pbs|>",
+    "<|lang:pei|>",
+    "<|lang:pia|>",
+    "<|lang:plo|>",
+    "<|lang:pls|>",
+    "<|lang:pmq|>",
+    "<|lang:pmz|>",
+    "<|lang:poe|>",
+    "<|lang:poi|>",
+    "<|lang:poi0|>",
+    "<|lang:pos|>",
+    "<|lang:ppi|>",
+    "<|lang:pps|>",
+    "<|lang:pt-BR|>",
+    "<|lang:pua|>",
+    "<|lang:quc|>",
+    "<|lang:sei|>",
+    "<|lang:spa|>",
+    "<|lang:st1*|>",
+    "<|lang:stp|>",
+    "<|lang:tac|>",
+    "<|lang:tar|>",
+    "<|lang:tar0|>",
+    "<|lang:tcf|>",
+    "<|lang:tee|>",
+    "<|lang:thh|>",
+    "<|lang:tku|>",
+    "<|lang:tl1*|>",
+    "<|lang:tla|>",
+    "<|lang:to1*|>",
+    "<|lang:toc|>",
+    "<|lang:toj|>",
+    "<|lang:too|>",
+    "<|lang:top|>",
+    "<|lang:tos|>",
+    "<|lang:tpl|>",
+    "<|lang:tpp|>",
+    "<|lang:tpt|>",
+    "<|lang:tpx|>",
+    "<|lang:tpx0|>",
+    "<|lang:tqt|>",
+    "<|lang:tr3*|>",
+    "<|lang:trc|>",
+    "<|lang:trs|>",
+    "<|lang:trs0|>",
+    "<|lang:ts4*|>",
+    "<|lang:ts9*|>",
+    "<|lang:tsz|>",
+    "<|lang:ttc|>",
+    "<|lang:tzh|>",
+    "<|lang:tzo|>",
+    "<|lang:unk|>",
+    "<|lang:var|>",
+    "<|lang:vmj|>",
+    "<|lang:vmm|>",
+    "<|lang:vmp|>",
+    "<|lang:vmy|>",
+    "<|lang:vmy0|>",
+    "<|lang:vmz|>",
+    "<|lang:wa1*|>",
+    "<|lang:xta|>",
+    "<|lang:xtd|>",
+    "<|lang:xti|>",
+    "<|lang:xtj|>",
+    "<|lang:xtm|>",
+    "<|lang:xtn|>",
+    "<|lang:yaq|>",
+    "<|lang:yua|>",
+    "<|lang:zaa|>",
+    "<|lang:zab|>",
+    "<|lang:zac|>",
+    "<|lang:zad|>",
+    "<|lang:zae|>",
+    "<|lang:zai|>",
+    "<|lang:zam|>",
+    "<|lang:zao|>",
+    "<|lang:zaq|>",
+    "<|lang:zaq0|>",
+    "<|lang:zar|>",
+    "<|lang:zas|>",
+    "<|lang:zav|>",
+    "<|lang:zaw|>",
+    "<|lang:zca|>",
+    "<|lang:zl3*|>",
+    "<|lang:zoc|>",
+    "<|lang:zoh|>",
+    "<|lang:zoh0|>",
+    "<|lang:zoq|>",
+    "<|lang:zoqa|>",
+    "<|lang:zor|>",
+    "<|lang:zos|>",
+    "<|lang:zpb|>",
+    "<|lang:zpc|>",
+    "<|lang:zpd|>",
+    "<|lang:zpf|>",
+    "<|lang:zpg|>",
+    "<|lang:zpi|>",
+    "<|lang:zpl|>",
+    "<|lang:zpm|>",
+    "<|lang:zpn|>",
+    "<|lang:zpo|>",
+    "<|lang:zpq|>",
+    "<|lang:zpr|>",
+    "<|lang:zpt|>",
+    "<|lang:zpu|>",
+    "<|lang:zpv|>",
+    "<|lang:zpz|>",
+    "<|lang:zsr|>",
+    "<|lang:ztg|>",
+    "<|lang:ztp|>",
+    "<|lang:ztq|>",
+    "<|lang:zts|>",
+    "<|reserved_0|>",
+    "<|reserved_1|>",
+    "<|reserved_2|>",
+    "<|reserved_3|>",
+    "<|reserved_4|>",
+    "<|reserved_5|>",
+    "<|reserved_6|>",
+    "<|reserved_7|>",
+    "<|reserved_8|>",
+    "<|reserved_9|>",
+    "<|reserved_10|>",
+    "<|reserved_11|>",
+    "<|reserved_12|>",
+    "<|reserved_13|>",
+    "<|reserved_14|>",
+    "<|reserved_15|>"
+  ]
+}

tokenizer_config.json

@@ -0,0 +1,289 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|startoftext|>",
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|tool_call_start|>",
+    "<|tool_call_end|>",
+    "<|tool_response_start|>",
+    "<|tool_response_end|>",
+    "<|reasoning_start|>",
+    "<|reasoning_end|>",
+    "<|fim_prefix|>",
+    "<|fim_middle|>",
+    "<|fim_suffix|>",
+    "<|fim_pad|>",
+    "<|repo_name|>",
+    "<|file_sep|>",
+    "<|lang:agu|>",
+    "<|lang:all|>",
+    "<|lang:am1*|>",
+    "<|lang:amu|>",
+    "<|lang:amu0|>",
+    "<|lang:azg|>",
+    "<|lang:azn|>",
+    "<|lang:azn0|>",
+    "<|lang:azz|>",
+    "<|lang:cac|>",
+    "<|lang:cak|>",
+    "<|lang:cco|>",
+    "<|lang:cco0|>",
+    "<|lang:chd|>",
+    "<|lang:chf|>",
+    "<|lang:chf0|>",
+    "<|lang:chj|>",
+    "<|lang:chq|>",
+    "<|lang:chz|>",
+    "<|lang:cle|>",
+    "<|lang:cnl|>",
+    "<|lang:cnt|>",
+    "<|lang:cok|>",
+    "<|lang:coz|>",
+    "<|lang:cpa|>",
+    "<|lang:crn|>",
+    "<|lang:cso|>",
+    "<|lang:ct2*|>",
+    "<|lang:cta|>",
+    "<|lang:cte|>",
+    "<|lang:ctl|>",
+    "<|lang:ctp|>",
+    "<|lang:ctp0|>",
+    "<|lang:ctu|>",
+    "<|lang:cuc|>",
+    "<|lang:cut|>",
+    "<|lang:cux|>",
+    "<|lang:de|>",
+    "<|lang:dih|>",
+    "<|lang:en-US|>",
+    "<|lang:eng|>",
+    "<|lang:es-MX|>",
+    "<|lang:esp|>",
+    "<|lang:fr|>",
+    "<|lang:hch|>",
+    "<|lang:hu2*|>",
+    "<|lang:hus|>",
+    "<|lang:huv|>",
+    "<|lang:hve|>",
+    "<|lang:hve0|>",
+    "<|lang:ixc|>",
+    "<|lang:jac|>",
+    "<|lang:jmx|>",
+    "<|lang:kak|>",
+    "<|lang:kek|>",
+    "<|lang:kic|>",
+    "<|lang:klb|>",
+    "<|lang:knj|>",
+    "<|lang:knja|>",
+    "<|lang:ku1*|>",
+    "<|lang:lac|>",
+    "<|lang:maa|>",
+    "<|lang:maj|>",
+    "<|lang:mam|>",
+    "<|lang:maq|>",
+    "<|lang:mat|>",
+    "<|lang:mau|>",
+    "<|lang:maz|>",
+    "<|lang:maz0|>",
+    "<|lang:mbz|>",
+    "<|lang:mbzl|>",
+    "<|lang:mco|>",
+    "<|lang:mco0|>",
+    "<|lang:meh|>",
+    "<|lang:mfy|>",
+    "<|lang:mhc|>",
+    "<|lang:mib|>",
+    "<|lang:mie|>",
+    "<|lang:mig|>",
+    "<|lang:mih|>",
+    "<|lang:mil|>",
+    "<|lang:mim|>",
+    "<|lang:mim0|>",
+    "<|lang:mio|>",
+    "<|lang:mir|>",
+    "<|lang:mit|>",
+    "<|lang:mix|>",
+    "<|lang:miy|>",
+    "<|lang:miz|>",
+    "<|lang:mjc|>",
+    "<|lang:mks|>",
+    "<|lang:ml2*|>",
+    "<|lang:ml8*|>",
+    "<|lang:mmc|>",
+    "<|lang:mop|>",
+    "<|lang:mpm|>",
+    "<|lang:mto|>",
+    "<|lang:mxb|>",
+    "<|lang:mxp|>",
+    "<|lang:mxq|>",
+    "<|lang:mxt|>",
+    "<|lang:mxv|>",
+    "<|lang:mxv0|>",
+    "<|lang:mxy|>",
+    "<|lang:mza|>",
+    "<|lang:mzi|>",
+    "<|lang:mzia|>",
+    "<|lang:naz|>",
+    "<|lang:nch|>",
+    "<|lang:ncj|>",
+    "<|lang:ncl|>",
+    "<|lang:ncx|>",
+    "<|lang:neq|>",
+    "<|lang:ngu|>",
+    "<|lang:nhe|>",
+    "<|lang:nhg|>",
+    "<|lang:nhi|>",
+    "<|lang:nhm|>",
+    "<|lang:nhn|>",
+    "<|lang:nhw|>",
+    "<|lang:nhx|>",
+    "<|lang:nhy|>",
+    "<|lang:nlv|>",
+    "<|lang:nlv0|>",
+    "<|lang:npl|>",
+    "<|lang:nsu|>",
+    "<|lang:ntp|>",
+    "<|lang:ocu|>",
+    "<|lang:ood|>",
+    "<|lang:opt|>",
+    "<|lang:ote|>",
+    "<|lang:otm|>",
+    "<|lang:otn|>",
+    "<|lang:otq|>",
+    "<|lang:ots|>",
+    "<|lang:ott|>",
+    "<|lang:pbs|>",
+    "<|lang:pei|>",
+    "<|lang:pia|>",
+    "<|lang:plo|>",
+    "<|lang:pls|>",
+    "<|lang:pmq|>",
+    "<|lang:pmz|>",
+    "<|lang:poe|>",
+    "<|lang:poi|>",
+    "<|lang:poi0|>",
+    "<|lang:pos|>",
+    "<|lang:ppi|>",
+    "<|lang:pps|>",
+    "<|lang:pt-BR|>",
+    "<|lang:pua|>",
+    "<|lang:quc|>",
+    "<|lang:sei|>",
+    "<|lang:spa|>",
+    "<|lang:st1*|>",
+    "<|lang:stp|>",
+    "<|lang:tac|>",
+    "<|lang:tar|>",
+    "<|lang:tar0|>",
+    "<|lang:tcf|>",
+    "<|lang:tee|>",
+    "<|lang:thh|>",
+    "<|lang:tku|>",
+    "<|lang:tl1*|>",
+    "<|lang:tla|>",
+    "<|lang:to1*|>",
+    "<|lang:toc|>",
+    "<|lang:toj|>",
+    "<|lang:too|>",
+    "<|lang:top|>",
+    "<|lang:tos|>",
+    "<|lang:tpl|>",
+    "<|lang:tpp|>",
+    "<|lang:tpt|>",
+    "<|lang:tpx|>",
+    "<|lang:tpx0|>",
+    "<|lang:tqt|>",
+    "<|lang:tr3*|>",
+    "<|lang:trc|>",
+    "<|lang:trs|>",
+    "<|lang:trs0|>",
+    "<|lang:ts4*|>",
+    "<|lang:ts9*|>",
+    "<|lang:tsz|>",
+    "<|lang:ttc|>",
+    "<|lang:tzh|>",
+    "<|lang:tzo|>",
+    "<|lang:unk|>",
+    "<|lang:var|>",
+    "<|lang:vmj|>",
+    "<|lang:vmm|>",
+    "<|lang:vmp|>",
+    "<|lang:vmy|>",
+    "<|lang:vmy0|>",
+    "<|lang:vmz|>",
+    "<|lang:wa1*|>",
+    "<|lang:xta|>",
+    "<|lang:xtd|>",
+    "<|lang:xti|>",
+    "<|lang:xtj|>",
+    "<|lang:xtm|>",
+    "<|lang:xtn|>",
+    "<|lang:yaq|>",
+    "<|lang:yua|>",
+    "<|lang:zaa|>",
+    "<|lang:zab|>",
+    "<|lang:zac|>",
+    "<|lang:zad|>",
+    "<|lang:zae|>",
+    "<|lang:zai|>",
+    "<|lang:zam|>",
+    "<|lang:zao|>",
+    "<|lang:zaq|>",
+    "<|lang:zaq0|>",
+    "<|lang:zar|>",
+    "<|lang:zas|>",
+    "<|lang:zav|>",
+    "<|lang:zaw|>",
+    "<|lang:zca|>",
+    "<|lang:zl3*|>",
+    "<|lang:zoc|>",
+    "<|lang:zoh|>",
+    "<|lang:zoh0|>",
+    "<|lang:zoq|>",
+    "<|lang:zoqa|>",
+    "<|lang:zor|>",
+    "<|lang:zos|>",
+    "<|lang:zpb|>",
+    "<|lang:zpc|>",
+    "<|lang:zpd|>",
+    "<|lang:zpf|>",
+    "<|lang:zpg|>",
+    "<|lang:zpi|>",
+    "<|lang:zpl|>",
+    "<|lang:zpm|>",
+    "<|lang:zpn|>",
+    "<|lang:zpo|>",
+    "<|lang:zpq|>",
+    "<|lang:zpr|>",
+    "<|lang:zpt|>",
+    "<|lang:zpu|>",
+    "<|lang:zpv|>",
+    "<|lang:zpz|>",
+    "<|lang:zsr|>",
+    "<|lang:ztg|>",
+    "<|lang:ztp|>",
+    "<|lang:ztq|>",
+    "<|lang:zts|>",
+    "<|reserved_0|>",
+    "<|reserved_1|>",
+    "<|reserved_2|>",
+    "<|reserved_3|>",
+    "<|reserved_4|>",
+    "<|reserved_5|>",
+    "<|reserved_6|>",
+    "<|reserved_7|>",
+    "<|reserved_8|>",
+    "<|reserved_9|>",
+    "<|reserved_10|>",
+    "<|reserved_11|>",
+    "<|reserved_12|>",
+    "<|reserved_13|>",
+    "<|reserved_14|>",
+    "<|reserved_15|>"
+  ],
+  "model_max_length": 4096,
+  "pad_token": "<|pad|>",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<|unk|>"
+}