Fix config hyperparameters for 80k architecture and tie embeddings

LICENSE.lic

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Quatum Technologies
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

READ.md

@@ -0,0 +1,63 @@
+---
+language:
+- en
+license: mit
+library_name: transformers
+pipeline_tag: text-generation
+tags:
+- tiny-models
+- custom-architecture
+- story-generation
+- experimental
+---
+
+# Spin-80k
+
+**Spin-80k** is a lightweight, 80k-parameter decoder-only language model built from scratch by **Quantech** to demonstrate custom Transformer architecture 
+
+---
+
+## Model Specifications
+
+* **Organization:** Quantech
+* **Architecture:** Custom Decoder-only Transformer
+* **Total Parameters:** ~80,112
+* **Layers:** 2
+* **Hidden Dimension ($d_{\text{model}}$):** 48
+* **Attention Heads:** 4
+* **Feed-Forward Dimension ($d_{\text{ff}}$):** 128
+* **Positional Encoding:** Rotary Position Embeddings (RoPE)
+* **Normalization:** RMSNorm ($\epsilon = 10^{-5}$)
+* **Activation:** SwiGLU
+* **Vocabulary:** 512 Byte-Pair Encoding (BPE) tokens
+* **Context Length:** 256 tokens
+
+---
+
+## Quickstart
+
+```python
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+repo_id = "Quantech/spin-80k"
+
+# Load Tokenizer & Model
+tokenizer = AutoTokenizer.from_pretrained(repo_id, trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(repo_id, trust_remote_code=True)
+model.eval()
+
+# ChatML Format
+prompt = "<|im_start|>user\nWrite a short story about a dog.<|im_end|>\n<|im_start|>assistant\n"
+inputs = tokenizer(prompt, return_tensors="pt")
+
+with torch.no_grad():
+    outputs = model.generate(
+        **inputs,
+        max_new_tokens=50,
+        temperature=0.7,
+        do_sample=True,
+        pad_token_id=tokenizer.eos_token_id
+    )
+
+print(tokenizer.decode(outputs[0]))

config.json

@@ -0,0 +1,15 @@
+{
+  "auto_map": {
+    "AutoConfig": "configuration_spin.SpinConfig",
+    "AutoModelForCausalLM": "modeling_spin.SpinForCausalLM"
+  },
+  "d_ff": 128,
+  "d_model": 48,
+  "max_seq_len": 256,
+  "model_type": "spin",
+  "n_heads": 4,
+  "n_layers": 2,
+  "norm_eps": 1e-05,
+  "transformers_version": "4.57.6",
+  "vocab_size": 512
+}

configuration_spin.py

@@ -0,0 +1,26 @@
+from transformers import PretrainedConfig
+
+
+class SpinConfig(PretrainedConfig):
+    model_type = "spin"
+
+    def __init__(
+        self,
+        vocab_size: int = 512,
+        max_seq_len: int = 256,
+        d_model: int = 48,
+        n_layers: int = 2,
+        n_heads: int = 4,
+        d_ff: int = 128,
+        norm_eps: float = 1e-5,
+        tie_word_embeddings: bool = True,
+        **kwargs,
+    ):
+        super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
+        self.vocab_size = vocab_size
+        self.max_seq_len = max_seq_len
+        self.d_model = d_model
+        self.n_layers = n_layers
+        self.n_heads = n_heads
+        self.d_ff = d_ff
+        self.norm_eps = norm_eps

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b743042e716a2dec638778a954a3f2c57ff58040dad09f6335de07cd30ced59
+size 957560

modeling_spin.py

@@ -0,0 +1,207 @@
+from configuration_spin import SpinConfig
+import math
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers import PreTrainedModel
+from transformers.modeling_outputs import CausalLMOutputWithPast
+
+class RMSNorm(nn.Module):
+
+    def __init__(self, dim: int, eps: float = 1e-5):
+        super().__init__()
+        self.eps = eps
+        self.weight = nn.Parameter(torch.ones(dim))
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        variance = x.pow(2).mean(-1, keepdim=True)
+        return x * torch.rsqrt(variance + self.eps) * self.weight
+
+def precompute_freqs_cis(dim: int, max_seq_len: int, theta: float = 10000.0):
+    freqs = 1.0 / (
+        theta ** (torch.arange(0, dim, 2)[: (dim // 2)].float() / dim)
+    )
+    t = torch.arange(max_seq_len, dtype=torch.float32)
+    freqs = torch.outer(t, freqs)
+    freqs_cos = torch.cos(freqs)
+    freqs_sin = torch.sin(freqs)
+    return freqs_cos, freqs_sin
+
+
+def apply_rotary_emb(
+    xq: torch.Tensor,
+    xk: torch.Tensor,
+    freqs_cos: torch.Tensor,
+    freqs_sin: torch.Tensor,
+):
+    
+    xq_r, xq_i = xq.float().reshape(*xq.shape[:-1], -1, 2).unbind(-1)
+    xk_r, xk_i = xk.float().reshape(*xk.shape[:-1], -1, 2).unbind(-1)
+
+   
+    freqs_cos = freqs_cos.unsqueeze(0).unsqueeze(2)
+    freqs_sin = freqs_sin.unsqueeze(0).unsqueeze(2)
+
+    xq_out_r = xq_r * freqs_cos - xq_i * freqs_sin
+    xq_out_i = xq_r * freqs_sin + xq_i * freqs_cos
+    xk_out_r = xk_r * freqs_cos - xk_i * freqs_sin
+    xk_out_i = xk_r * freqs_sin + xk_i * freqs_cos
+
+    xq_out = torch.stack([xq_out_r, xq_out_i], dim=-1).flatten(3)
+    xk_out = torch.stack([xk_out_r, xk_out_i], dim=-1).flatten(3)
+    return xq_out.type_as(xq), xk_out.type_as(xk)
+
+class SwiGLU(nn.Module):
+
+    def __init__(self, d_model: int, d_ff: int):
+        super().__init__()
+        self.w_gate = nn.Linear(d_model, d_ff, bias=False)
+        self.w_up = nn.Linear(d_model, d_ff, bias=False)
+        self.w_down = nn.Linear(d_ff, d_model, bias=False)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return self.w_down(F.silu(self.w_gate(x)) * self.w_up(x))
+
+class CausalSelfAttention(nn.Module):
+
+    def __init__(self, config: SpinConfig):
+        super().__init__()
+        self.n_heads = config.n_heads
+        self.head_dim = config.d_model // config.n_heads
+
+        self.q_proj = nn.Linear(config.d_model, config.d_model, bias=False)
+        self.k_proj = nn.Linear(config.d_model, config.d_model, bias=False)
+        self.v_proj = nn.Linear(config.d_model, config.d_model, bias=False)
+        self.out_proj = nn.Linear(config.d_model, config.d_model, bias=False)
+
+        mask = torch.full(
+            (config.max_seq_len, config.max_seq_len), float("-inf")
+        )
+        mask = torch.triu(mask, diagonal=1)
+        self.register_buffer("causal_mask", mask)
+
+    def forward(
+        self,
+        x: torch.Tensor,
+        freqs_cos: torch.Tensor,
+        freqs_sin: torch.Tensor,
+        kv_cache: tuple[torch.Tensor, torch.Tensor] | None = None,
+    ) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]:
+        B, T, C = x.shape
+
+        q = self.q_proj(x).view(B, T, self.n_heads, self.head_dim)
+        k = self.k_proj(x).view(B, T, self.n_heads, self.head_dim)
+        v = self.v_proj(x).view(B, T, self.n_heads, self.head_dim)
+
+        q, k = apply_rotary_emb(q, k, freqs_cos, freqs_sin)
+
+        q = q.transpose(1, 2)
+        k = k.transpose(1, 2)
+        v = v.transpose(1, 2)
+
+        if kv_cache is not None:
+            prev_k, prev_v = kv_cache
+            k = torch.cat([prev_k, k], dim=2)
+            v = torch.cat([prev_v, v], dim=2)
+        new_kv_cache = (k, v)
+
+        total_k_len = k.size(2)
+        scores = (q @ k.transpose(-2, -1)) / math.sqrt(self.head_dim)
+
+        if T > 1:
+            scores = scores + self.causal_mask[:T, :total_k_len]
+
+        attn_weights = F.softmax(scores, dim=-1)
+        out = attn_weights @ v
+        out = out.transpose(1, 2).contiguous().view(B, T, C)
+        return self.out_proj(out), new_kv_cache
+        
+class TransformerBlock(nn.Module):
+
+    def __init__(self, config: SpinConfig):
+        super().__init__()
+        self.attn_norm = RMSNorm(config.d_model, eps=config.norm_eps)
+        self.attn = CausalSelfAttention(config)
+        self.ffn_norm = RMSNorm(config.d_model, eps=config.norm_eps)
+        self.ffn = SwiGLU(config.d_model, config.d_ff)
+
+    def forward(
+        self,
+        x: torch.Tensor,
+        freqs_cos: torch.Tensor,
+        freqs_sin: torch.Tensor,
+        kv_cache: tuple[torch.Tensor, torch.Tensor] | None = None,
+    ) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]:
+        norm_x = self.attn_norm(x)
+        attn_out, next_kv = self.attn(
+            norm_x, freqs_cos, freqs_sin, kv_cache=kv_cache
+        )
+        x = x + attn_out
+        x = x + self.ffn(self.ffn_norm(x))
+        return x, next_kv
+
+
+class SpinForCausalLM(PreTrainedModel):
+    config_class = SpinConfig
+
+    def __init__(self, config: SpinConfig):
+        super().__init__(config)
+        self.config = config
+        self.tok_embeddings = nn.Embedding(config.vocab_size, config.d_model)
+        self.layers = nn.ModuleList(
+            [TransformerBlock(config) for _ in range(config.n_layers)]
+        )
+        self.norm = RMSNorm(config.d_model, eps=config.norm_eps)
+        self.lm_head = nn.Linear(config.d_model, config.vocab_size, bias=False)
+
+      
+        self.tok_embeddings.weight = self.lm_head.weight
+
+        head_dim = config.d_model // config.n_heads
+        freqs_cos, freqs_sin = precompute_freqs_cis(
+            head_dim, config.max_seq_len
+        )
+        self.register_buffer("freqs_cos", freqs_cos)
+        self.register_buffer("freqs_sin", freqs_sin)
+
+        self.post_init()
+
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        labels: torch.Tensor = None,
+        kv_caches=None,
+        start_pos: int = 0,
+        return_dict: bool = True,
+    ):
+        B, T = input_ids.shape
+        x = self.tok_embeddings(input_ids)
+
+        freqs_cos = self.freqs_cos[start_pos : start_pos + T]
+        freqs_sin = self.freqs_sin[start_pos : start_pos + T]
+
+        new_kv_caches = []
+        for i, layer in enumerate(self.layers):
+            cache_i = kv_caches[i] if kv_caches is not None else None
+            x, new_cache = layer(x, freqs_cos, freqs_sin, kv_cache=cache_i)
+            new_kv_caches.append(new_cache)
+
+        x = self.norm(x)
+        logits = self.lm_head(x)
+
+        loss = None
+        if labels is not None:
+            loss = F.cross_entropy(
+                logits.view(-1, self.config.vocab_size),
+                labels.view(-1),
+                ignore_index=-100,
+            )
+
+        if not return_dict:
+            return (logits, loss, new_kv_caches)
+
+        return CausalLMOutputWithPast(
+            loss=loss, logits=logits, past_key_values=new_kv_caches
+        )
+
+

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|im_start|>",
+  "eos_token": "<|im_end|>",
+  "pad_token": "<|pad|>"
+}

tokenizer.json

@@ -0,0 +1,1601 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "<|unk|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "<|pad|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "<|bos|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "<|eos|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 4,
+      "content": "<|im_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 5,
+      "content": "<|im_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": null,
+  "pre_tokenizer": {
+    "type": "ByteLevel",
+    "add_prefix_space": false,
+    "trim_offsets": true,
+    "use_regex": true
+  },
+  "post_processor": null,
+  "decoder": {
+    "type": "ByteLevel",
+    "add_prefix_space": true,
+    "trim_offsets": true,
+    "use_regex": true
+  },
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": "<|unk|>",
+    "continuing_subword_prefix": null,
+    "end_of_word_suffix": null,
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "ignore_merges": false,
+    "vocab": {
+      "<|unk|>": 0,
+      "<|pad|>": 1,
+      "<|bos|>": 2,
+      "<|eos|>": 3,
+      "<|im_start|>": 4,
+      "<|im_end|>": 5,
+      "!": 6,
+      "\"": 7,
+      "#": 8,
+      "$": 9,
+      "%": 10,
+      "&": 11,
+      "'": 12,
+      "(": 13,
+      ")": 14,
+      "*": 15,
+      "+": 16,
+      ",": 17,
+      "-": 18,
+      ".": 19,
+      "/": 20,
+      "0": 21,
+      "1": 22,
+      "2": 23,
+      "3": 24,
+      "4": 25,
+      "5": 26,
+      "6": 27,
+      "7": 28,
+      "8": 29,
+      "9": 30,
+      ":": 31,
+      ";": 32,
+      "<": 33,
+      "=": 34,
+      ">": 35,
+      "?": 36,
+      "@": 37,
+      "A": 38,
+      "B": 39,
+      "C": 40,
+      "D": 41,
+      "E": 42,
+      "F": 43,
+      "G": 44,
+      "H": 45,
+      "I": 46,
+      "J": 47,
+      "K": 48,
+      "L": 49,
+      "M": 50,
+      "N": 51,
+      "O": 52,
+      "P": 53,
+      "Q": 54,
+      "R": 55,
+      "S": 56,
+      "T": 57,
+      "U": 58,
+      "V": 59,
+      "W": 60,
+      "X": 61,
+      "Y": 62,
+      "Z": 63,
+      "[": 64,
+      "\\": 65,
+      "]": 66,
+      "^": 67,
+      "_": 68,
+      "`": 69,
+      "a": 70,
+      "b": 71,
+      "c": 72,
+      "d": 73,
+      "e": 74,
+      "f": 75,
+      "g": 76,
+      "h": 77,
+      "i": 78,
+      "j": 79,
+      "k": 80,
+      "l": 81,
+      "m": 82,
+      "n": 83,
+      "o": 84,
+      "p": 85,
+      "q": 86,
+      "r": 87,
+      "s": 88,
+      "t": 89,
+      "u": 90,
+      "v": 91,
+      "w": 92,
+      "x": 93,
+      "y": 94,
+      "z": 95,
+      "{": 96,
+      "|": 97,
+      "}": 98,
+      "~": 99,
+      "¡": 100,
+      "¢": 101,
+      "£": 102,
+      "¤": 103,
+      "¥": 104,
+      "¦": 105,
+      "§": 106,
+      "¨": 107,
+      "©": 108,
+      "ª": 109,
+      "«": 110,
+      "¬": 111,
+      "®": 112,
+      "¯": 113,
+      "°": 114,
+      "±": 115,
+      "²": 116,
+      "³": 117,
+      "´": 118,
+      "µ": 119,
+      "¶": 120,
+      "·": 121,
+      "¸": 122,
+      "¹": 123,
+      "º": 124,
+      "»": 125,
+      "¼": 126,
+      "½": 127,
+      "¾": 128,
+      "¿": 129,
+      "À": 130,
+      "Á": 131,
+      "Â": 132,
+      "Ã": 133,
+      "Ä": 134,
+      "Å": 135,
+      "Æ": 136,
+      "Ç": 137,
+      "È": 138,
+      "É": 139,
+      "Ê": 140,
+      "Ë": 141,
+      "Ì": 142,
+      "Í": 143,
+      "Î": 144,
+      "Ï": 145,
+      "Ð": 146,
+      "Ñ": 147,
+      "Ò": 148,
+      "Ó": 149,
+      "Ô": 150,
+      "Õ": 151,
+      "Ö": 152,
+      "×": 153,
+      "Ø": 154,
+      "Ù": 155,
+      "Ú": 156,
+      "Û": 157,
+      "Ü": 158,
+      "Ý": 159,
+      "Þ": 160,
+      "ß": 161,
+      "à": 162,
+      "á": 163,
+      "â": 164,
+      "ã": 165,
+      "ä": 166,
+      "å": 167,
+      "æ": 168,
+      "ç": 169,
+      "è": 170,
+      "é": 171,
+      "ê": 172,
+      "ë": 173,
+      "ì": 174,
+      "í": 175,
+      "î": 176,
+      "ï": 177,
+      "ð": 178,
+      "ñ": 179,
+      "ò": 180,
+      "ó": 181,
+      "ô": 182,
+      "õ": 183,
+      "ö": 184,
+      "÷": 185,
+      "ø": 186,
+      "ù": 187,
+      "ú": 188,
+      "û": 189,
+      "ü": 190,
+      "ý": 191,
+      "��": 192,
+      "ÿ": 193,
+      "Ā": 194,
+      "ā": 195,
+      "Ă": 196,
+      "ă": 197,
+      "Ą": 198,
+      "ą": 199,
+      "Ć": 200,
+      "ć": 201,
+      "Ĉ": 202,
+      "ĉ": 203,
+      "Ċ": 204,
+      "ċ": 205,
+      "Č": 206,
+      "č": 207,
+      "Ď": 208,
+      "ď": 209,
+      "Đ": 210,
+      "đ": 211,
+      "Ē": 212,
+      "ē": 213,
+      "Ĕ": 214,
+      "ĕ": 215,
+      "Ė": 216,
+      "ė": 217,
+      "Ę": 218,
+      "ę": 219,
+      "Ě": 220,
+      "ě": 221,
+      "Ĝ": 222,
+      "ĝ": 223,
+      "Ğ": 224,
+      "ğ": 225,
+      "Ġ": 226,
+      "ġ": 227,
+      "Ģ": 228,
+      "ģ": 229,
+      "Ĥ": 230,
+      "ĥ": 231,
+      "Ħ": 232,
+      "ħ": 233,
+      "Ĩ": 234,
+      "ĩ": 235,
+      "Ī": 236,
+      "ī": 237,
+      "Ĭ": 238,
+      "ĭ": 239,
+      "Į": 240,
+      "į": 241,
+      "İ": 242,
+      "ı": 243,
+      "IJ": 244,
+      "ij": 245,
+      "Ĵ": 246,
+      "ĵ": 247,
+      "Ķ": 248,
+      "ķ": 249,
+      "ĸ": 250,
+      "Ĺ": 251,
+      "ĺ": 252,
+      "Ļ": 253,
+      "ļ": 254,
+      "Ľ": 255,
+      "ľ": 256,
+      "Ŀ": 257,
+      "ŀ": 258,
+      "Ł": 259,
+      "ł": 260,
+      "Ń": 261,
+      "he": 262,
+      "Ġt": 263,
+      "Ġa": 264,
+      "Ġs": 265,
+      "Ġw": 266,
+      "nd": 267,
+      "Ġthe": 268,
+      "ed": 269,
+      "Ġand": 270,
+      "Ġto": 271,
+      "Ġb": 272,
+      "in": 273,
+      "Ġh": 274,
+      "Ġwa": 275,
+      "re": 276,
+      "Ġf": 277,
+      "ou": 278,
+      "it": 279,
+      "Ġl": 280,
+      "Ġc": 281,
+      "Ġd": 282,
+      "Ġhe": 283,
+      "er": 284,
+      "Ġm": 285,
+      "Ġp": 286,
+      "Ġwas": 287,
+      "ĠT": 288,
+      "om": 289,
+      "Ġo": 290,
+      "ay": 291,
+      "is": 292,
+      "ar": 293,
+      "il": 294,
+      "ing": 295,
+      "Ġg": 296,
+      "id": 297,
+      "at": 298,
+      "en": 299,
+      "Ġn": 300,
+      "Ġsa": 301,
+      "Ġha": 302,
+      "ĠS": 303,
+      "ĠThe": 304,
+      "an": 305,
+      "im": 306,
+      "or": 307,
+      "on": 308,
+      "Ġit": 309,
+      "Ġth": 310,
+      "ll": 311,
+      "le": 312,
+      "ĠH": 313,
+      "Ġher": 314,
+      "et": 315,
+      "ot": 316,
+      "ir": 317,
+      "ĠShe": 318,
+      "ĠHe": 319,
+      "es": 320,
+      "Ġin": 321,
+      "ow": 322,
+      "ut": 323,
+      "ver": 324,
+      "ĠThey": 325,
+      "ck": 326,
+      "ily": 327,
+      "Ġu": 328,
+      "Ġe": 329,
+      "Ġy": 330,
+      "ld": 331,
+      "oo": 332,
+      "Ġsaid": 333,
+      "ig": 334,
+      "Ġ\"": 335,
+      "Ġr": 336,
+      "am": 337,
+      "Ġbe": 338,
+      "ĠI": 339,
+      "Ġst": 340,
+      "Lily": 341,
+      "ith": 342,
+      "ke": 343,
+      "ce": 344,
+      "pp": 345,
+      "ve": 346,
+      "Ġwith": 347,
+      "Ġshe": 348,
+      "On": 349,
+      "Ġon": 350,
+      "Ġyou": 351,
+      "ri": 352,
+      "ked": 353,
+      "Ġpl": 354,
+      "Ġhis": 355,
+      "Ġof": 356,
+      "nt": 357,
+      "Ġso": 358,
+      "ad": 359,
+      "Ġday": 360,
+      "very": 361,
+      "Ġthat": 362,
+      "ĠLily": 363,
+      "Ġplay": 364,
+      "Ġup": 365,
+      "Ġwe": 366,
+      "Ġmom": 367,
+      "Ġhad": 368,
+      "Ġthey": 369,
+      "st": 370,
+      "ĠB": 371,
+      "el": 372,
+      "Ġfor": 373,
+      "un": 374,
+      "my": 375,
+      "'s": 376,
+      "ould": 377,
+      "Ġli": 378,
+      "The": 379,
+      "Ġhapp": 380,
+      "ent": 381,
+      "ch": 382,
+      "itt": 383,
+      "Ġwant": 384,
+      "her": 385,
+      "Ġsh": 386,
+      "out": 387,
+      "Ġdo": 388,
+      "Ġnot": 389,
+      "end": 390,
+      "ome": 391,
+      "all": 392,
+      "se": 393,
+      "Ġvery": 394,
+      "ound": 395,
+      "ly": 396,
+      "ime": 397,
+      "ittle": 398,
+      "Ġk": 399,
+      "Ġtime": 400,
+      "Ġwh": 401,
+      "Ġlittle": 402,
+      "Ġthere": 403,
+      "Ġne": 404,
+      "Ġis": 405,
+      "Ġbig": 406,
+      "ht": 407,
+      "Ġhappy": 408,
+      "al": 409,
+      "ĠM": 410,
+      "Ġre": 411,
+      "Ġsm": 412,
+      "ĠĊ": 413,
+      "riend": 414,
+      "Ġfriend": 415,
+      "ĠIt": 416,
+      "Ġsaw": 417,
+      "Ġbut": 418,
+      ".\"": 419,
+      "ra": 420,
+      "Ġan": 421,
+      "Ġas": 422,
+      "ry": 423,
+      "One": 424,
+      "ack": 425,
+      "ide": 426,
+      "ake": 427,
+      "ved": 428,
+      "'t": 429,
+      "!\"": 430,
+      "ug": 431,
+      "ter": 432,
+      "Ġlo": 433,
+      "Ġloo": 434,
+      "Once": 435,
+      "Ġwere": 436,
+      "ĠTim": 437,
+      "Ġse": 438,
+      "ore": 439,
+      "Ġbo": 440,
+      "Ġtoo": 441,
+      "Ġhim": 442,
+      "Ġgo": 443,
+      "Ġare": 444,
+      "ec": 445,
+      "Ġupon": 446,
+      "ill": 447,
+      "irl": 448,
+      "Ġwanted": 449,
+      "Ġthem": 450,
+      "ard": 451,
+      "Ġj": 452,
+      "Ġgirl": 453,
+      "Ġout": 454,
+      "Ġtheir": 455,
+      "Ġat": 456,
+      "ind": 457,
+      "way": 458,
+      "ur": 459,
+      "fu": 460,
+      "Ġdid": 461,
+      "ĠA": 462,
+      "Ġsp": 463,
+      "Ġsmil": 464,
+      "Ġcould": 465,
+      "hen": 466,
+      "Ġhave": 467,
+      "ain": 468,
+      "ĠBen": 469,
+      "Ġwent": 470,
+      "hed": 471,
+      "Ġnam": 472,
+      "Ġcan": 473,
+      "art": 474,
+      "rom": 475,
+      "lp": 476,
+      "ic": 477,
+      "Ġex": 478,
+      "round": 479,
+      "Ġfriends": 480,
+      "?\"": 481,
+      "Ġhelp": 482,
+      "ood": 483,
+      "ful": 484,
+      "Ġkn": 485,
+      "ĠJ": 486,
+      "Ġall": 487,
+      "hing": 488,
+      "ight": 489,
+      "um": 490,
+      "ark": 491,
+      "Ġfun": 492,
+      "one": 493,
+      "Ġsay": 494,
+      "Ġback": 495,
+      "You": 496,
+      "Ġnamed": 497,
+      "are": 498,
+      "Ġcl": 499,
+      "ĠTom": 500,
+      "Ġno": 501,
+      "Ġle": 502,
+      "Ġloved": 503,
+      "op": 504,
+      "elt": 505,
+      "ĠTimmy": 506,
+      "ĠOne": 507,
+      "oug": 508,
+      "Ġsc": 509,
+      "Ġfelt": 510,
+      "Ġtoy": 511
+    },
+    "merges": [
+      [
+        "h",
+        "e"
+      ],
+      [
+        "Ġ",
+        "t"
+      ],
+      [
+        "Ġ",
+        "a"
+      ],
+      [
+        "Ġ",
+        "s"
+      ],
+      [
+        "Ġ",
+        "w"
+      ],
+      [
+        "n",
+        "d"
+      ],
+      [
+        "Ġt",
+        "he"
+      ],
+      [
+        "e",
+        "d"
+      ],
+      [
+        "Ġa",
+        "nd"
+      ],
+      [
+        "Ġt",
+        "o"
+      ],
+      [
+        "Ġ",
+        "b"
+      ],
+      [
+        "i",
+        "n"
+      ],
+      [
+        "Ġ",
+        "h"
+      ],
+      [
+        "Ġw",
+        "a"
+      ],
+      [
+        "r",
+        "e"
+      ],
+      [
+        "Ġ",
+        "f"
+      ],
+      [
+        "o",
+        "u"
+      ],
+      [
+        "i",
+        "t"
+      ],
+      [
+        "Ġ",
+        "l"
+      ],
+      [
+        "Ġ",
+        "c"
+      ],
+      [
+        "Ġ",
+        "d"
+      ],
+      [
+        "Ġ",
+        "he"
+      ],
+      [
+        "e",
+        "r"
+      ],
+      [
+        "Ġ",
+        "m"
+      ],
+      [
+        "Ġ",
+        "p"
+      ],
+      [
+        "Ġwa",
+        "s"
+      ],
+      [
+        "Ġ",
+        "T"
+      ],
+      [
+        "o",
+        "m"
+      ],
+      [
+        "Ġ",
+        "o"
+      ],
+      [
+        "a",
+        "y"
+      ],
+      [
+        "i",
+        "s"
+      ],
+      [
+        "a",
+        "r"
+      ],
+      [
+        "i",
+        "l"
+      ],
+      [
+        "in",
+        "g"
+      ],
+      [
+        "Ġ",
+        "g"
+      ],
+      [
+        "i",
+        "d"
+      ],
+      [
+        "a",
+        "t"
+      ],
+      [
+        "e",
+        "n"
+      ],
+      [
+        "Ġ",
+        "n"
+      ],
+      [
+        "Ġs",
+        "a"
+      ],
+      [
+        "Ġh",
+        "a"
+      ],
+      [
+        "Ġ",
+        "S"
+      ],
+      [
+        "ĠT",
+        "he"
+      ],
+      [
+        "a",
+        "n"
+      ],
+      [
+        "i",
+        "m"
+      ],
+      [
+        "o",
+        "r"
+      ],
+      [
+        "o",
+        "n"
+      ],
+      [
+        "Ġ",
+        "it"
+      ],
+      [
+        "Ġt",
+        "h"
+      ],
+      [
+        "l",
+        "l"
+      ],
+      [
+        "l",
+        "e"
+      ],
+      [
+        "Ġ",
+        "H"
+      ],
+      [
+        "Ġhe",
+        "r"
+      ],
+      [
+        "e",
+        "t"
+      ],
+      [
+        "o",
+        "t"
+      ],
+      [
+        "i",
+        "r"
+      ],
+      [
+        "ĠS",
+        "he"
+      ],
+      [
+        "ĠH",
+        "e"
+      ],
+      [
+        "e",
+        "s"
+      ],
+      [
+        "Ġ",
+        "in"
+      ],
+      [
+        "o",
+        "w"
+      ],
+      [
+        "u",
+        "t"
+      ],
+      [
+        "v",
+        "er"
+      ],
+      [
+        "ĠThe",
+        "y"
+      ],
+      [
+        "c",
+        "k"
+      ],
+      [
+        "il",
+        "y"
+      ],
+      [
+        "Ġ",
+        "u"
+      ],
+      [
+        "Ġ",
+        "e"
+      ],
+      [
+        "Ġ",
+        "y"
+      ],
+      [
+        "l",
+        "d"
+      ],
+      [
+        "o",
+        "o"
+      ],
+      [
+        "Ġsa",
+        "id"
+      ],
+      [
+        "i",
+        "g"
+      ],
+      [
+        "Ġ",
+        "\""
+      ],
+      [
+        "Ġ",
+        "r"
+      ],
+      [
+        "a",
+        "m"
+      ],
+      [
+        "Ġb",
+        "e"
+      ],
+      [
+        "Ġ",
+        "I"
+      ],
+      [
+        "Ġs",
+        "t"
+      ],
+      [
+        "L",
+        "ily"
+      ],
+      [
+        "it",
+        "h"
+      ],
+      [
+        "k",
+        "e"
+      ],
+      [
+        "c",
+        "e"
+      ],
+      [
+        "p",
+        "p"
+      ],
+      [
+        "v",
+        "e"
+      ],
+      [
+        "Ġw",
+        "ith"
+      ],
+      [
+        "Ġs",
+        "he"
+      ],
+      [
+        "O",
+        "n"
+      ],
+      [
+        "Ġo",
+        "n"
+      ],
+      [
+        "Ġy",
+        "ou"
+      ],
+      [
+        "r",
+        "i"
+      ],
+      [
+        "k",
+        "ed"
+      ],
+      [
+        "Ġp",
+        "l"
+      ],
+      [
+        "Ġh",
+        "is"
+      ],
+      [
+        "Ġo",
+        "f"
+      ],
+      [
+        "n",
+        "t"
+      ],
+      [
+        "Ġs",
+        "o"
+      ],
+      [
+        "a",
+        "d"
+      ],
+      [
+        "Ġd",
+        "ay"
+      ],
+      [
+        "ver",
+        "y"
+      ],
+      [
+        "Ġth",
+        "at"
+      ],
+      [
+        "Ġ",
+        "Lily"
+      ],
+      [
+        "Ġpl",
+        "ay"
+      ],
+      [
+        "Ġu",
+        "p"
+      ],
+      [
+        "Ġw",
+        "e"
+      ],
+      [
+        "Ġm",
+        "om"
+      ],
+      [
+        "Ġha",
+        "d"
+      ],
+      [
+        "Ġthe",
+        "y"
+      ],
+      [
+        "s",
+        "t"
+      ],
+      [
+        "Ġ",
+        "B"
+      ],
+      [
+        "e",
+        "l"
+      ],
+      [
+        "Ġf",
+        "or"
+      ],
+      [
+        "u",
+        "n"
+      ],
+      [
+        "m",
+        "y"
+      ],
+      [
+        "'",
+        "s"
+      ],
+      [
+        "ou",
+        "ld"
+      ],
+      [
+        "Ġl",
+        "i"
+      ],
+      [
+        "T",
+        "he"
+      ],
+      [
+        "Ġha",
+        "pp"
+      ],
+      [
+        "en",
+        "t"
+      ],
+      [
+        "c",
+        "h"
+      ],
+      [
+        "it",
+        "t"
+      ],
+      [
+        "Ġwa",
+        "nt"
+      ],
+      [
+        "he",
+        "r"
+      ],
+      [
+        "Ġs",
+        "h"
+      ],
+      [
+        "ou",
+        "t"
+      ],
+      [
+        "Ġd",
+        "o"
+      ],
+      [
+        "Ġn",
+        "ot"
+      ],
+      [
+        "e",
+        "nd"
+      ],
+      [
+        "om",
+        "e"
+      ],
+      [
+        "a",
+        "ll"
+      ],
+      [
+        "s",
+        "e"
+      ],
+      [
+        "Ġ",
+        "very"
+      ],
+      [
+        "ou",
+        "nd"
+      ],
+      [
+        "l",
+        "y"
+      ],
+      [
+        "im",
+        "e"
+      ],
+      [
+        "itt",
+        "le"
+      ],
+      [
+        "Ġ",
+        "k"
+      ],
+      [
+        "Ġt",
+        "ime"
+      ],
+      [
+        "Ġw",
+        "h"
+      ],
+      [
+        "Ġl",
+        "ittle"
+      ],
+      [
+        "Ġthe",
+        "re"
+      ],
+      [
+        "Ġn",
+        "e"
+      ],
+      [
+        "Ġ",
+        "is"
+      ],
+      [
+        "Ġb",
+        "ig"
+      ],
+      [
+        "h",
+        "t"
+      ],
+      [
+        "Ġhapp",
+        "y"
+      ],
+      [
+        "a",
+        "l"
+      ],
+      [
+        "Ġ",
+        "M"
+      ],
+      [
+        "Ġ",
+        "re"
+      ],
+      [
+        "Ġs",
+        "m"
+      ],
+      [
+        "Ġ",
+        "Ċ"
+      ],
+      [
+        "ri",
+        "end"
+      ],
+      [
+        "Ġf",
+        "riend"
+      ],
+      [
+        "ĠI",
+        "t"
+      ],
+      [
+        "Ġsa",
+        "w"
+      ],
+      [
+        "Ġb",
+        "ut"
+      ],
+      [
+        ".",
+        "\""
+      ],
+      [
+        "r",
+        "a"
+      ],
+      [
+        "Ġa",
+        "n"
+      ],
+      [
+        "Ġa",
+        "s"
+      ],
+      [
+        "r",
+        "y"
+      ],
+      [
+        "On",
+        "e"
+      ],
+      [
+        "a",
+        "ck"
+      ],
+      [
+        "id",
+        "e"
+      ],
+      [
+        "a",
+        "ke"
+      ],
+      [
+        "v",
+        "ed"
+      ],
+      [
+        "'",
+        "t"
+      ],
+      [
+        "!",
+        "\""
+      ],
+      [
+        "u",
+        "g"
+      ],
+      [
+        "t",
+        "er"
+      ],
+      [
+        "Ġl",
+        "o"
+      ],
+      [
+        "Ġl",
+        "oo"
+      ],
+      [
+        "On",
+        "ce"
+      ],
+      [
+        "Ġwe",
+        "re"
+      ],
+      [
+        "ĠT",
+        "im"
+      ],
+      [
+        "Ġs",
+        "e"
+      ],
+      [
+        "o",
+        "re"
+      ],
+      [
+        "Ġb",
+        "o"
+      ],
+      [
+        "Ġto",
+        "o"
+      ],
+      [
+        "Ġh",
+        "im"
+      ],
+      [
+        "Ġg",
+        "o"
+      ],
+      [
+        "Ġa",
+        "re"
+      ],
+      [
+        "e",
+        "c"
+      ],
+      [
+        "Ġup",
+        "on"
+      ],
+      [
+        "il",
+        "l"
+      ],
+      [
+        "ir",
+        "l"
+      ],
+      [
+        "Ġwant",
+        "ed"
+      ],
+      [
+        "Ġthe",
+        "m"
+      ],
+      [
+        "ar",
+        "d"
+      ],
+      [
+        "Ġ",
+        "j"
+      ],
+      [
+        "Ġg",
+        "irl"
+      ],
+      [
+        "Ġ",
+        "out"
+      ],
+      [
+        "Ġthe",
+        "ir"
+      ],
+      [
+        "Ġa",
+        "t"
+      ],
+      [
+        "i",
+        "nd"
+      ],
+      [
+        "w",
+        "ay"
+      ],
+      [
+        "u",
+        "r"
+      ],
+      [
+        "f",
+        "u"
+      ],
+      [
+        "Ġd",
+        "id"
+      ],
+      [
+        "Ġ",
+        "A"
+      ],
+      [
+        "Ġs",
+        "p"
+      ],
+      [
+        "Ġsm",
+        "il"
+      ],
+      [
+        "Ġc",
+        "ould"
+      ],
+      [
+        "he",
+        "n"
+      ],
+      [
+        "Ġha",
+        "ve"
+      ],
+      [
+        "a",
+        "in"
+      ],
+      [
+        "ĠB",
+        "en"
+      ],
+      [
+        "Ġw",
+        "ent"
+      ],
+      [
+        "he",
+        "d"
+      ],
+      [
+        "Ġn",
+        "am"
+      ],
+      [
+        "Ġc",
+        "an"
+      ],
+      [
+        "ar",
+        "t"
+      ],
+      [
+        "r",
+        "om"
+      ],
+      [
+        "l",
+        "p"
+      ],
+      [
+        "i",
+        "c"
+      ],
+      [
+        "Ġe",
+        "x"
+      ],
+      [
+        "r",
+        "ound"
+      ],
+      [
+        "Ġfriend",
+        "s"
+      ],
+      [
+        "?",
+        "\""
+      ],
+      [
+        "Ġhe",
+        "lp"
+      ],
+      [
+        "oo",
+        "d"
+      ],
+      [
+        "fu",
+        "l"
+      ],
+      [
+        "Ġk",
+        "n"
+      ],
+      [
+        "Ġ",
+        "J"
+      ],
+      [
+        "Ġa",
+        "ll"
+      ],
+      [
+        "h",
+        "ing"
+      ],
+      [
+        "ig",
+        "ht"
+      ],
+      [
+        "u",
+        "m"
+      ],
+      [
+        "ar",
+        "k"
+      ],
+      [
+        "Ġf",
+        "un"
+      ],
+      [
+        "on",
+        "e"
+      ],
+      [
+        "Ġs",
+        "ay"
+      ],
+      [
+        "Ġb",
+        "ack"
+      ],
+      [
+        "Y",
+        "ou"
+      ],
+      [
+        "Ġnam",
+        "ed"
+      ],
+      [
+        "a",
+        "re"
+      ],
+      [
+        "Ġc",
+        "l"
+      ],
+      [
+        "ĠT",
+        "om"
+      ],
+      [
+        "Ġn",
+        "o"
+      ],
+      [
+        "Ġl",
+        "e"
+      ],
+      [
+        "Ġlo",
+        "ved"
+      ],
+      [
+        "o",
+        "p"
+      ],
+      [
+        "el",
+        "t"
+      ],
+      [
+        "ĠTim",
+        "my"
+      ],
+      [
+        "Ġ",
+        "One"
+      ],
+      [
+        "ou",
+        "g"
+      ],
+      [
+        "Ġs",
+        "c"
+      ],
+      [
+        "Ġf",
+        "elt"
+      ],
+      [
+        "Ġto",
+        "y"
+      ]
+    ]
+  }
+}

tokenizer_config.json

@@ -0,0 +1,59 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|unk|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|bos|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<|eos|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|im_start|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|pad|>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}