Upload tokenizer/__init__.py with huggingface_hub

tokenizer/__init__.py

@@ -0,0 +1,23 @@
+"""JSCoder tokenizer package: byte-level BPE + PyTorch-friendly wrapper."""
+
+from .special_tokens import (
+    EOT,
+    FIM_MIDDLE,
+    FIM_PAD,
+    FIM_PREFIX,
+    FIM_SUFFIX,
+    PAD,
+    SPECIAL_TOKENS,
+)
+from .tokenizer import JSCoderTokenizer
+
+__all__ = [
+    "JSCoderTokenizer",
+    "SPECIAL_TOKENS",
+    "EOT",
+    "PAD",
+    "FIM_PREFIX",
+    "FIM_MIDDLE",
+    "FIM_SUFFIX",
+    "FIM_PAD",
+]