Spaces:

quickgrid
/

Tokenizer-Visualizer

Running

App Files Files Community

quickgrid commited on 5 days ago

Commit

76938dc

verified ·

1 Parent(s): ea5426c

Update index.html

Browse files

Files changed (1) hide show

index.html +1491 -271

index.html CHANGED Viewed

@@ -1,302 +1,1522 @@
 <!DOCTYPE html>
 <html lang="en">
 <head>
-  <meta charset="UTF-8">
-  <meta name="viewport" content="width=device-width, initial-scale=1.0">
-  <title>Universal Tokenizer Visualizer</title>
-  <script src="https://cdn.tailwindcss.com"></script>
-  <link href="https://cdn.jsdelivr.net/npm/@fontsource/inter@5.0.16/index.min.css" rel="stylesheet">
   <style>
-    body { font-family: 'Inter', sans-serif; background: linear-gradient(135deg, #0f172a 0%, #1e293b 100%); color: #e2e8f0; }
-    .glass { background: rgba(30, 41, 59, 0.6); backdrop-filter: blur(12px); border: 1px solid rgba(255,255,255,0.08); }
-    .token-chip { transition: all 0.15s cubic-bezier(0.4, 0, 0.2, 1); cursor: default; position: relative; user-select: none; }
-    .token-chip:hover { transform: translateY(-3px) scale(1.05); z-index: 20; box-shadow: 0 8px 20px -4px rgba(0,0,0,0.5); }
-    .token-chip::after {
-      content: attr(data-tooltip);
-      position: absolute; bottom: 110%; left: 50%; transform: translateX(-50%) translateY(8px);
-      background: #0f172a; color: #cbd5e1; padding: 8px 12px; border-radius: 8px;
-      font-size: 0.75rem; line-height: 1.4; white-space: pre-wrap; max-width: 260px;
-      opacity: 0; pointer-events: none; transition: all 0.2s; border: 1px solid #334155;
-      box-shadow: 0 4px 12px rgba(0,0,0,0.4); z-index: 30; text-align: left;
-    }
-    .token-chip:hover::after { opacity: 1; transform: translateX(-50%) translateY(0); }
-    .spinner { width: 20px; height: 20px; border: 2px solid rgba(255,255,255,0.2); border-radius: 50%; border-top-color: #60a5fa; animation: spin 0.8s linear infinite; }
     @keyframes spin { to { transform: rotate(360deg); } }
-    ::-webkit-scrollbar { width: 8px; }
-    ::-webkit-scrollbar-track { background: #1e293b; }
-    ::-webkit-scrollbar-thumb { background: #475569; border-radius: 4px; }
-    .fade-in { animation: fadeIn 0.3s ease-out forwards; opacity: 0; }
-    @keyframes fadeIn { to { opacity: 1; transform: translateY(0); } }
-    .glow-btn { box-shadow: 0 0 20px rgba(59, 130, 246, 0.3); }
-    .glow-btn:hover { box-shadow: 0 0 30px rgba(59, 130, 246, 0.5); }
   </style>
 </head>
-<body class="min-h-screen flex flex-col items-center justify-start p-4 md:p-8">
-  <header class="w-full max-w-5xl mb-8 text-center md:text-left">
-    <h1 class="text-4xl font-bold bg-gradient-to-r from-cyan-400 via-blue-500 to-purple-500 bg-clip-text text-transparent tracking-tight">
-      🧩 Universal Tokenizer Visualizer
-    </h1>
-    <p class="text-slate-400 mt-2 text-sm md:text-base">
-      Load any HuggingFace tokenizer directly in your browser. 100% client-side, zero GPU cost.
-    </p>
-  </header>
-  <main class="w-full max-w-5xl grid grid-cols-1 lg:grid-cols-12 gap-6">
-    <!-- Control Panel -->
-    <section class="lg:col-span-4 glass rounded-2xl p-5 flex flex-col gap-4 shadow-2xl">
-      <div>
-        <label class="block text-sm font-medium text-slate-300 mb-1.5">HuggingFace Model ID</label>
-        <input type="text" id="modelId" value="gpt2"
-          class="w-full bg-slate-900/80 border border-slate-600 rounded-lg px-3 py-2.5 text-sm focus:outline-none focus:ring-2 focus:ring-blue-500/50 transition placeholder-slate-500"
-          placeholder="e.g., gpt2, meta-llama/Llama-3-8B">
-        <p class="text-xs text-slate-500 mt-1">Must contain <code class="bg-slate-800 px-1 rounded text-slate-300">tokenizer.json</code></p>
       </div>
-      <div class="flex-grow">
-        <label class="block text-sm font-medium text-slate-300 mb-1.5">Input Text</label>
-        <textarea id="inputText" rows="7"
-          class="w-full bg-slate-900/80 border border-slate-600 rounded-lg px-3 py-2.5 text-sm focus:outline-none focus:ring-2 focus:ring-blue-500/50 resize-y transition placeholder-slate-500"
-          placeholder="Type or paste text to visualize..."></textarea>
       </div>
-      <div class="flex flex-col gap-2 bg-slate-900/40 p-3 rounded-lg border border-slate-700/50">
-        <label class="flex items-center gap-2 cursor-pointer select-none">
-          <input type="checkbox" id="addSpecial" class="w-4 h-4 rounded bg-slate-800 border-slate-600 text-blue-500 focus:ring-blue-500" checked>
-          <span class="text-sm text-slate-300">Include special tokens (BOS/EOS)</span>
-        </label>
-        <label class="flex items-center gap-2 cursor-pointer select-none">
-          <input type="checkbox" id="showBytes" class="w-4 h-4 rounded bg-slate-800 border-slate-600 text-blue-500 focus:ring-blue-500">
-          <span class="text-sm text-slate-300">Show byte values for specials</span>
-        </label>
       </div>
-      <button id="tokenizeBtn" class="w-full bg-blue-600 hover:bg-blue-500 active:scale-[0.98] text-white font-semibold py-2.5 px-4 rounded-lg transition flex items-center justify-center gap-2 disabled:opacity-50 disabled:cursor-not-allowed glow-btn">
-        <span>✨ Tokenize</span>
-      </button>
-      <div id="status" class="text-sm text-slate-400 h-6 flex items-center gap-2 truncate px-1"></div>
-    </section>
-    <!-- Visualization Panel -->
-    <section class="lg:col-span-8 glass rounded-2xl p-5 flex flex-col shadow-2xl min-h-[500px]">
-      <div class="flex flex-wrap justify-between items-center mb-4 pb-3 border-b border-slate-700/50 gap-3">
-        <h2 class="text-lg font-semibold text-slate-100">Token Breakdown</h2>
-        <div class="flex flex-wrap gap-3 text-sm">
-          <div class="flex items-center gap-1.5 bg-slate-800 px-3 py-1.5 rounded-md border border-slate-700">
-            <span class="w-2 h-2 rounded-full bg-blue-400"></span>
-            <span class="text-slate-300">Tokens: <b id="statTokens" class="text-white">0</b></span>
-          </div>
-          <div class="flex items-center gap-1.5 bg-slate-800 px-3 py-1.5 rounded-md border border-slate-700">
-            <span class="w-2 h-2 rounded-full bg-purple-400"></span>
-            <span class="text-slate-300">Chars: <b id="statChars" class="text-white">0</b></span>
-          </div>
-          <div class="flex items-center gap-1.5 bg-slate-800 px-3 py-1.5 rounded-md border border-slate-700">
-            <span class="w-2 h-2 rounded-full bg-pink-400"></span>
-            <span class="text-slate-300">Ratio: <b id="statRatio" class="text-white">0.00</b></span>
-          </div>
         </div>
       </div>
-      <div id="outputGrid" class="flex flex-wrap gap-2 overflow-y-auto p-3 content-start flex-grow min-h-[350px] bg-slate-900/30 rounded-xl border border-slate-800/40">
-        <div class="w-full h-full flex flex-col items-center justify-center text-slate-500 select-none py-12">
-          <svg class="w-14 h-14 mb-4 opacity-30" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="1.5" d="M13 10V3L4 14h7v7l9-11h-7z"></path></svg>
-          <p class="mb-2">Enter text and click <span class="text-blue-400 font-medium">Tokenize</span> to visualize.</p>
-          <p class="text-xs opacity-60">Try <code>gpt2</code>, <code>bert-base-uncased</code>, or <code>Qwen/Qwen2.5-7B</code></p>
         </div>
       </div>
-    </section>
   </main>
-  <footer class="mt-10 text-center text-slate-600 text-xs flex flex-col md:flex-row items-center gap-2 pb-6">
-    <span>Powered by <code class="bg-slate-800 px-1.5 py-0.5 rounded text-blue-400">@huggingface/transformers@3</code></span>
-    <span>•</span>
-    <span>Runs entirely in-browser via WebAssembly</span>
-    <span>•</span>
-    <a href="https://github.com/huggingface/transformers.js" target="_blank" class="text-blue-400 hover:underline">Docs</a>
   </footer>
-  <script type="module">
-    // Import transformers.js from CDN (ESM compatible)
-    import { AutoTokenizer } from 'https://esm.sh/@huggingface/transformers@3.0.2';
-    const COLORS = [
-      '#3b82f6', '#8b5cf6', '#ec4899', '#f59e0b', '#10b981',
-      '#06b6d4', '#6366f1', '#f97316', '#84cc16', '#14b8a6', '#f43f5e', '#a855f7'
-    ];
-    const els = {
-      modelId: document.getElementById('modelId'),
-      inputText: document.getElementById('inputText'),
-      btn: document.getElementById('tokenizeBtn'),
-      output: document.getElementById('outputGrid'),
-      status: document.getElementById('status'),
-      statTokens: document.getElementById('statTokens'),
-      statChars: document.getElementById('statChars'),
-      statRatio: document.getElementById('statRatio'),
-      addSpecial: document.getElementById('addSpecial'),
-      showBytes: document.getElementById('showBytes')
-    };
-    let tokenizerCache = null;
-    function getColor(idx) { return COLORS[idx % COLORS.length]; }
-    function setStatus(msg, type = 'info') {
-      els.status.textContent = msg;
-      els.status.className = `text-sm h-6 flex items-center gap-2 truncate px-1 ${
-        type === 'error' ? 'text-red-400' :
-        type === 'success' ? 'text-green-400' :
-        'text-slate-400'
-      }`;
-    }
-    async function loadTokenizer(modelId) {
-      if (tokenizerCache && tokenizerCache._id === modelId) return tokenizerCache.tokenizer;
-      setStatus('📥 Fetching tokenizer configuration...', 'info');
-      try {
-        // Progress callback for better UX on large tokenizers
-        const tokenizer = await AutoTokenizer.from_pretrained(modelId, {
-          progress_callback: (p) => {
-            if (p.status === 'loading' || p.status === 'downloading') {
-              const pct = Math.round((p.loaded / p.total) * 100);
-              setStatus(`📥 ${pct}% ${p.file || 'loading'}`, 'info');
-            }
-          }
-        });
-        tokenizerCache = { _id: modelId, tokenizer };
-        return tokenizer;
-      } catch (err) {
-        console.error(err);
-        throw new Error(`Failed to load tokenizer. Check Model ID, network, or repo visibility. (${err.message})`);
-      }
     }
-    function getBackend(tokenizer) {
-      // transformers.js v3 exposes backend_tokenizer
-      // Fallback to common property names if structure changes
-      return tokenizer.backend_tokenizer ?? tokenizer.tokenizer ?? tokenizer;
-    }
-    els.btn.addEventListener('click', async () => {
-      const modelId = els.modelId.value.trim();
-      const text = els.inputText.value;
-      const addSpecial = els.addSpecial.checked;
-      const showBytes = els.showBytes.checked;
-      if (!modelId) return setStatus('⚠️ Please enter a Model ID', 'error');
-      if (!text) return setStatus('⚠️ Please enter text to tokenize', 'error');
-      els.btn.disabled = true;
-      els.btn.innerHTML = '<div class="spinner"></div> <span class="ml-2">Loading...</span>';
-      els.output.innerHTML = '';
-      setStatus('🔄 Initializing tokenizer...');
-      try {
-        const tokenizer = await loadTokenizer(modelId);
-        setStatus('⚡ Running tokenization...', 'info');
-        const backend = getBackend(tokenizer);
-        if (!backend || typeof backend.encode !== 'function') {
-          throw new Error('Backend tokenizer encoding method not found. This model format might not be supported in-browser.');
-        }
-        // encode returns { tokens, ids, offsets, ... }
-        const result = backend.encode(text, { add_special_tokens: addSpecial });
-        renderTokens(result.tokens, result.ids, result.offsets, text, showBytes);
-        setStatus(`✨ Done • ${result.ids.length} tokens loaded`, 'success');
-      } catch (err) {
-        console.error('Tokenization error:', err);
-        setStatus(`❌ ${err.message}`, 'error');
-        els.output.innerHTML = `
-          <div class="w-full text-center py-12">
-            <div class="text-red-400 font-medium mb-2">Tokenization Failed</div>
-            <div class="text-slate-500 text-sm max-w-md mx-auto space-y-2">
-              <p>${err.message}</p>
-              <div class="bg-slate-800/50 p-3 rounded border border-slate-700 text-left mt-4">
-                <p class="text-slate-400 mb-1">Troubleshooting:</p>
-                <ul class="list-disc list-inside space-y-1 text-xs text-slate-500">
-                  <li>Try a base model first: <code>gpt2</code>, <code>bert-base-uncased</code></li>
-                  <li>Ensure <code>tokenizer.json</code> exists in the repo</li>
-                  <li>Custom tokenizers (e.g., DeepSeek-V3) may need specific config</li>
-                  <li>Check browser console for network/CORS errors</li>
-                </ul>
-              </div>
-            </div>
-          </div>`;
-      } finally {
-        els.btn.disabled = false;
-        els.btn.innerHTML = '<span>✨ Tokenize</span>';
-      }
     });
-    function renderTokens(tokens, ids, offsets, originalText, showBytes) {
-      els.output.innerHTML = '';
-      const count = ids.length;
-      els.statTokens.textContent = count;
-      els.statChars.textContent = originalText.length;
-      els.statRatio.textContent = originalText.length > 0 ? (count / originalText.length).toFixed(2) : '0.00';
-      tokens.forEach((tok, i) => {
-        const [start, end] = offsets[i] || [0, 0];
-        const chunk = originalText.slice(start, end);
-        const color = getColor(i);
-        const id = ids[i];
-        const chip = document.createElement('div');
-        chip.className = 'token-chip rounded-md px-2 py-1.5 text-sm font-mono border select-none fade-in';
-        chip.style.animationDelay = `${Math.min(i * 0.02, 2)}s`; // Cap animation delay
-        chip.style.animationDelay = `${i * 0.015}s`;
-        chip.style.backgroundColor = `${color}15`;
-        chip.style.borderColor = `${color}80`;
-        chip.style.color = color;
-        // Tooltip details
-        const cleanChunk = chunk.replace(/"/g, '\\"').replace(/\n/g, '\\n').replace(/\t/g, '\\t');
-        const byteInfo = showBytes ? `\nBytes: ${[...chunk].map(c => '0x'+c.charCodeAt(0).toString(16).padStart(2,'0')).join(' ')}` : '';
-        chip.dataset.tooltip = `ID: ${id}\nToken: "${tok}"\nOffset: [${start}, ${end})\nText: "${cleanChunk}"${byteInfo}`;
-        // Visual handling of special/invisible tokens
-        let displayText = tok;
-        if (tok.startsWith('<') && tok.endsWith('>')) {
-          chip.classList.add('border-dashed', 'opacity-80');
-        } else if (chunk === '') {
-          displayText = '␣';
-        } else if (chunk === '\n') {
-          displayText = '↵';
-        } else if (chunk === '\t') {
-          displayText = '→';
         }
-        chip.innerHTML = `<span class="font-medium">${displayText}</span><span class="opacity-60 text-[10px] ml-1.5 align-top font-sans">${id}</span>`;
-        // Interactive highlight
-        chip.addEventListener('click', () => {
-          document.querySelectorAll('.token-chip').forEach(c => c.style.boxShadow = 'none');
-          chip.style.boxShadow = `0 0 0 2px ${color}, 0 4px 12px ${color}40`;
-          // Optional: auto-scroll to text or highlight in textarea
-        });
-        els.output.appendChild(chip);
-      });
-    }
-    // Keyboard shortcut: Ctrl/Cmd + Enter
-    document.addEventListener('keydown', (e) => {
-      if ((e.ctrlKey || e.metaKey) && e.key === 'Enter') {
-        els.btn.click();
       }
     });
-    // Init demo
-    window.addEventListener('load', () => {
-      if (!els.inputText.value) {
-        els.inputText.value = "Tokenization is the bridge between human language and AI. It breaks text into numerical pieces! 🚀";
-      }
-    });
-  </script>
-</body>
-</html>

 <!DOCTYPE html>
 <html lang="en">
 <head>
+  <meta charset="UTF-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>TokenLens — LLM Tokenizer Playground</title>
+  <meta name="description" content="Visualize how large language models tokenize text. Powered by Transformers.js, runs entirely in your browser." />
+  <link rel="preconnect" href="https://fonts.googleapis.com" />
+  <link href="https://fonts.googleapis.com/css2?family=Bricolage+Grotesque:opsz,wght@12..96,300;12..96,400;12..96,500;12..96,600;12..96,700;12..96,800&family=JetBrains+Mono:wght@300;400;500;700&family=DM+Sans:wght@300;400;500&display=swap" rel="stylesheet" />
   <style>
+    /* ─── Design Tokens ─────────────────────────────────── */
+    :root {
+      --bg:       #060b14;
+      --bg2:      #0b1220;
+      --bg3:      #101828;
+      --bg4:      #162035;
+      --border:   #1a2d4a;
+      --border2:  #243d60;
+      --glow:     #1f3d6e;
+      --text:     #dce8f8;
+      --text2:    #7899c0;
+      --text3:    #3d5a80;
+      --accent:   #4d9ef5;
+      --accent2:  #8b6af5;
+      --green:    #34d89a;
+      --amber:    #f5a623;
+      --red:      #f55577;
+      /* Token palette — 14 vivid colors for dark bg */
+      --t0:  #ff8080; --t0b:  rgba(255,128,128,.18);
+      --t1:  #ffb84d; --t1b:  rgba(255,184, 77,.18);
+      --t2:  #ffe066; --t2b:  rgba(255,224,102,.18);
+      --t3:  #7aed91; --t3b:  rgba(122,237,145,.18);
+      --t4:  #4ddfc0; --t4b:  rgba( 77,223,192,.18);
+      --t5:  #56c8f5; --t5b:  rgba( 86,200,245,.18);
+      --t6:  #748ef8; --t6b:  rgba(116,142,248,.18);
+      --t7:  #c484f8; --t7b:  rgba(196,132,248,.18);
+      --t8:  #f57cd4; --t8b:  rgba(245,124,212,.18);
+      --t9:  #fa8072; --t9b:  rgba(250,128,114,.18);
+      --t10: #8be08b; --t10b: rgba(139,224,139,.18);
+      --t11: #f0c040; --t11b: rgba(240,192, 64,.18);
+      --t12: #60d4e0; --t12b: rgba( 96,212,224,.18);
+      --t13: #e89060; --t13b: rgba(232,144, 96,.18);
+    }
+    /* ─── Reset ─────────────────────────────────────────── */
+    *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
+    html { scroll-behavior: smooth; }
+    body {
+      background: var(--bg);
+      color: var(--text);
+      font-family: 'DM Sans', sans-serif;
+      min-height: 100vh;
+      display: flex;
+      flex-direction: column;
+      overflow-x: hidden;
+    }
+    /* ─── Background FX ─────────────────────────────────── */
+    #bg-canvas {
+      position: fixed;
+      inset: 0;
+      pointer-events: none;
+      z-index: 0;
+    }
+    .bg-gradient {
+      position: fixed;
+      inset: 0;
+      pointer-events: none;
+      z-index: 0;
+      background:
+        radial-gradient(ellipse 80% 50% at 20% 10%, rgba(77,158,245,.06) 0%, transparent 70%),
+        radial-gradient(ellipse 60% 40% at 80% 90%, rgba(139,106,245,.05) 0%, transparent 60%),
+        radial-gradient(ellipse 40% 30% at 60% 50%, rgba(52,216,154,.03) 0%, transparent 60%);
+    }
+    .dot-grid {
+      position: fixed;
+      inset: 0;
+      pointer-events: none;
+      z-index: 0;
+      background-image: radial-gradient(circle, rgba(77,158,245,.12) 1px, transparent 1px);
+      background-size: 36px 36px;
+      mask-image: radial-gradient(ellipse 100% 100% at 50% 50%, black 30%, transparent 80%);
+    }
+    /* ─── Layout ─────────────────────────────────────────── */
+    #app {
+      position: relative;
+      z-index: 1;
+      display: flex;
+      flex-direction: column;
+      min-height: 100vh;
+    }
+    /* ─── Header ─────────────────────────────────────────── */
+    header {
+      display: flex;
+      align-items: center;
+      justify-content: space-between;
+      padding: 0 32px;
+      height: 64px;
+      border-bottom: 1px solid var(--border);
+      background: rgba(6,11,20,.85);
+      backdrop-filter: blur(20px);
+      position: sticky;
+      top: 0;
+      z-index: 100;
+    }
+    .logo {
+      display: flex;
+      align-items: center;
+      gap: 10px;
+      text-decoration: none;
+      color: var(--text);
+    }
+    .logo-hex {
+      width: 34px;
+      height: 34px;
+      background: linear-gradient(135deg, var(--accent), var(--accent2));
+      clip-path: polygon(50% 0%, 93% 25%, 93% 75%, 50% 100%, 7% 75%, 7% 25%);
+      display: flex;
+      align-items: center;
+      justify-content: center;
+      font-size: 14px;
+      font-family: 'JetBrains Mono', monospace;
+      font-weight: 700;
+      color: white;
+    }
+    .logo-name {
+      font-family: 'Bricolage Grotesque', sans-serif;
+      font-size: 20px;
+      font-weight: 700;
+      letter-spacing: -0.5px;
+      background: linear-gradient(135deg, #dce8f8 40%, var(--accent));
+      -webkit-background-clip: text;
+      -webkit-text-fill-color: transparent;
+      background-clip: text;
+    }
+    .logo-tag {
+      font-size: 10px;
+      font-family: 'JetBrains Mono', monospace;
+      color: var(--text3);
+      background: var(--bg3);
+      border: 1px solid var(--border);
+      padding: 2px 6px;
+      border-radius: 4px;
+      letter-spacing: .5px;
+    }
+    .header-right {
+      display: flex;
+      align-items: center;
+      gap: 16px;
+    }
+    .header-badge {
+      display: flex;
+      align-items: center;
+      gap: 6px;
+      font-size: 12px;
+      color: var(--text2);
+      font-family: 'JetBrains Mono', monospace;
+    }
+    .header-badge .dot {
+      width: 7px;
+      height: 7px;
+      border-radius: 50%;
+      background: var(--green);
+      box-shadow: 0 0 8px var(--green);
+      animation: pulse-dot 2s ease-in-out infinite;
+    }
+    @keyframes pulse-dot {
+      0%,100% { opacity: 1; }
+      50% { opacity: .4; }
+    }
+    /* ─── Model Selector Bar ─────────────────────────────── */
+    .model-bar {
+      padding: 16px 32px;
+      border-bottom: 1px solid var(--border);
+      background: rgba(11,18,32,.7);
+      backdrop-filter: blur(12px);
+    }
+    .model-bar-label {
+      font-size: 11px;
+      font-family: 'JetBrains Mono', monospace;
+      color: var(--text3);
+      letter-spacing: 1.5px;
+      text-transform: uppercase;
+      margin-bottom: 10px;
+    }
+    .model-tabs {
+      display: flex;
+      flex-wrap: wrap;
+      gap: 8px;
+      align-items: center;
+    }
+    .model-tab {
+      display: flex;
+      flex-direction: column;
+      padding: 8px 14px;
+      border: 1px solid var(--border);
+      border-radius: 10px;
+      background: var(--bg2);
+      cursor: pointer;
+      transition: all 0.2s ease;
+      position: relative;
+      overflow: hidden;
+      min-width: 110px;
+    }
+    .model-tab::before {
+      content: '';
+      position: absolute;
+      inset: 0;
+      background: linear-gradient(135deg, var(--accent), var(--accent2));
+      opacity: 0;
+      transition: opacity 0.2s;
+    }
+    .model-tab:hover {
+      border-color: var(--border2);
+      transform: translateY(-1px);
+    }
+    .model-tab.active {
+      border-color: var(--accent);
+      box-shadow: 0 0 0 1px var(--accent), 0 0 20px rgba(77,158,245,.15);
+    }
+    .model-tab.active::before { opacity: .08; }
+    .model-tab-name {
+      font-family: 'Bricolage Grotesque', sans-serif;
+      font-size: 13px;
+      font-weight: 600;
+      color: var(--text);
+      position: relative;
+    }
+    .model-tab-org {
+      font-size: 10px;
+      color: var(--text2);
+      font-family: 'JetBrains Mono', monospace;
+      position: relative;
+      margin-top: 1px;
+    }
+    .model-tab-vocab {
+      font-size: 10px;
+      color: var(--text3);
+      font-family: 'JetBrains Mono', monospace;
+      position: relative;
+    }
+    .model-org-dot {
+      width: 6px;
+      height: 6px;
+      border-radius: 50%;
+      display: inline-block;
+      margin-right: 4px;
+      position: relative;
+      top: -1px;
+    }
+    /* Custom model row */
+    .custom-model-row {
+      display: flex;
+      align-items: center;
+      gap: 10px;
+      margin-top: 12px;
+    }
+    .custom-model-row label {
+      font-size: 11px;
+      color: var(--text2);
+      font-family: 'JetBrains Mono', monospace;
+      white-space: nowrap;
+    }
+    .custom-input {
+      flex: 1;
+      max-width: 380px;
+      background: var(--bg2);
+      border: 1px solid var(--border);
+      border-radius: 8px;
+      color: var(--text);
+      font-family: 'JetBrains Mono', monospace;
+      font-size: 13px;
+      padding: 7px 12px;
+      outline: none;
+      transition: border-color 0.2s;
+    }
+    .custom-input:focus { border-color: var(--accent); }
+    .custom-input::placeholder { color: var(--text3); }
+    .btn {
+      padding: 7px 16px;
+      border-radius: 8px;
+      border: 1px solid var(--border2);
+      background: linear-gradient(135deg, rgba(77,158,245,.15), rgba(139,106,245,.15));
+      color: var(--accent);
+      font-family: 'DM Sans', sans-serif;
+      font-size: 13px;
+      font-weight: 500;
+      cursor: pointer;
+      transition: all 0.2s;
+      white-space: nowrap;
+    }
+    .btn:hover {
+      background: linear-gradient(135deg, rgba(77,158,245,.25), rgba(139,106,245,.25));
+      border-color: var(--accent);
+    }
+    .btn:active { transform: scale(.97); }
+    /* ─── Main Split ─────────────────────────────────────── */
+    main {
+      flex: 1;
+      display: grid;
+      grid-template-columns: 1fr 1fr;
+      gap: 0;
+      min-height: 0;
+    }
+    /* ─── Left Panel (Input) ─────────────────────────────── */
+    .input-panel {
+      border-right: 1px solid var(--border);
+      display: flex;
+      flex-direction: column;
+      padding: 0;
+    }
+    .panel-header {
+      padding: 16px 24px 12px;
+      border-bottom: 1px solid var(--border);
+      display: flex;
+      align-items: center;
+      justify-content: space-between;
+    }
+    .panel-title {
+      font-family: 'Bricolage Grotesque', sans-serif;
+      font-size: 14px;
+      font-weight: 600;
+      color: var(--text2);
+      letter-spacing: .3px;
+      display: flex;
+      align-items: center;
+      gap: 8px;
+    }
+    .panel-title-icon {
+      width: 20px;
+      height: 20px;
+      background: var(--bg4);
+      border: 1px solid var(--border);
+      border-radius: 5px;
+      display: flex;
+      align-items: center;
+      justify-content: center;
+      font-size: 11px;
+    }
+    .sample-btns {
+      display: flex;
+      gap: 6px;
+    }
+    .sample-btn {
+      font-size: 11px;
+      padding: 4px 10px;
+      border-radius: 6px;
+      border: 1px solid var(--border);
+      background: var(--bg2);
+      color: var(--text2);
+      cursor: pointer;
+      font-family: 'DM Sans', sans-serif;
+      transition: all .15s;
+    }
+    .sample-btn:hover {
+      border-color: var(--border2);
+      color: var(--text);
+    }
+    #input-area {
+      flex: 1;
+      width: 100%;
+      background: transparent;
+      border: none;
+      outline: none;
+      resize: none;
+      color: var(--text);
+      font-family: 'DM Sans', sans-serif;
+      font-size: 15px;
+      line-height: 1.7;
+      padding: 20px 24px;
+      min-height: 220px;
+    }
+    #input-area::placeholder { color: var(--text3); }
+    .char-counter {
+      padding: 8px 24px;
+      border-top: 1px solid var(--border);
+      font-size: 11px;
+      font-family: 'JetBrains Mono', monospace;
+      color: var(--text3);
+      text-align: right;
+    }
+    /* ─── Right Panel (Output) ───────────────────────────── */
+    .output-panel {
+      display: flex;
+      flex-direction: column;
+      overflow: hidden;
+    }
+    /* Stats row */
+    .stats-row {
+      display: grid;
+      grid-template-columns: repeat(4, 1fr);
+      border-bottom: 1px solid var(--border);
+    }
+    .stat-card {
+      padding: 16px 20px;
+      border-right: 1px solid var(--border);
+      position: relative;
+      overflow: hidden;
+    }
+    .stat-card:last-child { border-right: none; }
+    .stat-card::after {
+      content: '';
+      position: absolute;
+      bottom: 0;
+      left: 0;
+      right: 0;
+      height: 2px;
+      background: linear-gradient(90deg, transparent, var(--accent), transparent);
+      opacity: 0;
+      transition: opacity .3s;
+    }
+    .stat-card.highlight::after { opacity: 1; }
+    .stat-label {
+      font-size: 10px;
+      font-family: 'JetBrains Mono', monospace;
+      color: var(--text3);
+      text-transform: uppercase;
+      letter-spacing: 1px;
+      margin-bottom: 6px;
+    }
+    .stat-value {
+      font-family: 'Bricolage Grotesque', sans-serif;
+      font-size: 26px;
+      font-weight: 700;
+      color: var(--text);
+      line-height: 1;
+      transition: all .3s;
+    }
+    .stat-card:nth-child(1) .stat-value { color: var(--accent); }
+    .stat-card:nth-child(2) .stat-value { color: var(--green); }
+    .stat-card:nth-child(3) .stat-value { color: var(--amber); }
+    .stat-card:nth-child(4) .stat-value { color: var(--accent2); }
+    .stat-sub {
+      font-size: 10px;
+      color: var(--text3);
+      font-family: 'JetBrains Mono', monospace;
+      margin-top: 3px;
+    }
+    /* View toggle */
+    .view-toggle {
+      display: flex;
+      padding: 12px 20px;
+      border-bottom: 1px solid var(--border);
+      gap: 4px;
+      align-items: center;
+      justify-content: space-between;
+    }
+    .toggle-group {
+      display: flex;
+      gap: 4px;
+      background: var(--bg2);
+      border: 1px solid var(--border);
+      border-radius: 8px;
+      padding: 3px;
+    }
+    .toggle-btn {
+      padding: 5px 14px;
+      border-radius: 6px;
+      border: none;
+      background: transparent;
+      color: var(--text2);
+      font-family: 'DM Sans', sans-serif;
+      font-size: 12px;
+      font-weight: 500;
+      cursor: pointer;
+      transition: all .15s;
+    }
+    .toggle-btn.active {
+      background: var(--bg4);
+      color: var(--text);
+      box-shadow: 0 1px 4px rgba(0,0,0,.3);
+    }
+    .special-toggle {
+      display: flex;
+      align-items: center;
+      gap: 8px;
+      font-size: 12px;
+      color: var(--text2);
+    }
+    .toggle-switch {
+      width: 32px;
+      height: 18px;
+      background: var(--bg4);
+      border: 1px solid var(--border);
+      border-radius: 9px;
+      cursor: pointer;
+      position: relative;
+      transition: background .2s;
+    }
+    .toggle-switch::after {
+      content: '';
+      position: absolute;
+      width: 12px;
+      height: 12px;
+      border-radius: 50%;
+      background: var(--text3);
+      top: 2px;
+      left: 2px;
+      transition: all .2s;
+    }
+    .toggle-switch.on { background: rgba(77,158,245,.3); border-color: var(--accent); }
+    .toggle-switch.on::after { left: 16px; background: var(--accent); }
+    /* Token Display */
+    .token-display {
+      flex: 1;
+      overflow-y: auto;
+      padding: 20px;
+      scrollbar-width: thin;
+      scrollbar-color: var(--border) transparent;
+    }
+    .placeholder-msg {
+      display: flex;
+      flex-direction: column;
+      align-items: center;
+      justify-content: center;
+      height: 200px;
+      gap: 16px;
+      color: var(--text3);
+    }
+    .placeholder-icon {
+      font-size: 40px;
+      filter: grayscale(1) opacity(.3);
+    }
+    .placeholder-msg p {
+      font-family: 'JetBrains Mono', monospace;
+      font-size: 13px;
+      text-align: center;
+    }
+    /* ─── Token Visualization Views ───────────────────────── */
+    /* TEXT VIEW — inline colored token spans */
+    .token-text-view {
+      font-family: 'JetBrains Mono', monospace;
+      font-size: 14px;
+      line-height: 2.2;
+      word-break: break-all;
+    }
+    .tok {
+      display: inline;
+      border-radius: 4px;
+      padding: 1px 0;
+      cursor: default;
+      transition: filter .15s;
+      position: relative;
+    }
+    .tok:hover { filter: brightness(1.3); }
+    .tok-tooltip {
+      display: none;
+      position: absolute;
+      bottom: 110%;
+      left: 50%;
+      transform: translateX(-50%);
+      background: var(--bg4);
+      border: 1px solid var(--border2);
+      border-radius: 6px;
+      padding: 5px 8px;
+      font-size: 11px;
+      white-space: nowrap;
+      z-index: 50;
+      pointer-events: none;
+      box-shadow: 0 4px 20px rgba(0,0,0,.5);
+    }
+    .tok:hover .tok-tooltip { display: block; }
+    .tok-tooltip-id { color: var(--accent); font-weight: 700; }
+    .tok-tooltip-text { color: var(--text2); }
+    .tok-space::before { content: '·'; opacity: .3; }
+    .tok-newline::before { content: '↵'; opacity: .5; }
+    /* ID VIEW — grid of token cards */
+    .token-id-view {
+      display: flex;
+      flex-wrap: wrap;
+      gap: 6px;
+    }
+    .tok-id-card {
+      display: flex;
+      flex-direction: column;
+      align-items: center;
+      border-radius: 8px;
+      overflow: hidden;
+      border: 1px solid;
+      cursor: default;
+      transition: transform .15s, box-shadow .15s;
+      min-width: 52px;
+    }
+    .tok-id-card:hover {
+      transform: translateY(-2px);
+      box-shadow: 0 4px 16px rgba(0,0,0,.4);
+    }
+    .tok-id-top {
+      padding: 3px 6px;
+      font-family: 'JetBrains Mono', monospace;
+      font-size: 11px;
+      font-weight: 500;
+      width: 100%;
+      text-align: center;
+      border-bottom: 1px solid rgba(255,255,255,.08);
+    }
+    .tok-id-bottom {
+      padding: 2px 6px 3px;
+      font-family: 'JetBrains Mono', monospace;
+      font-size: 9px;
+      color: rgba(255,255,255,.4);
+      width: 100%;
+      text-align: center;
+    }
+    /* PROBABILITY VIEW placeholder */
+    .token-split-view {
+      display: flex;
+      flex-direction: column;
+      gap: 3px;
+    }
+    .tok-split-row {
+      display: flex;
+      align-items: stretch;
+      border-radius: 6px;
+      overflow: hidden;
+      border: 1px solid;
+      font-family: 'JetBrains Mono', monospace;
+      font-size: 12px;
+    }
+    .tok-split-idx {
+      width: 38px;
+      text-align: center;
+      padding: 5px 4px;
+      font-size: 10px;
+      color: rgba(255,255,255,.3);
+      border-right: 1px solid rgba(255,255,255,.06);
+      display: flex;
+      align-items: center;
+      justify-content: center;
+    }
+    .tok-split-text {
+      flex: 1;
+      padding: 5px 8px;
+      font-size: 13px;
+    }
+    .tok-split-id {
+      padding: 5px 8px;
+      font-size: 11px;
+      color: rgba(255,255,255,.45);
+      border-left: 1px solid rgba(255,255,255,.06);
+      display: flex;
+      align-items: center;
+    }
+    /* ─── Loading Overlay ────────────────────────────────── */
+    #loading-overlay {
+      position: fixed;
+      inset: 0;
+      background: rgba(6,11,20,.92);
+      backdrop-filter: blur(8px);
+      z-index: 1000;
+      display: flex;
+      flex-direction: column;
+      align-items: center;
+      justify-content: center;
+      gap: 24px;
+      transition: opacity .4s;
+    }
+    #loading-overlay.hidden { opacity: 0; pointer-events: none; }
+    .loading-spinner {
+      width: 56px;
+      height: 56px;
+      position: relative;
+    }
+    .loading-spinner::before,
+    .loading-spinner::after {
+      content: '';
+      position: absolute;
+      border-radius: 50%;
+      border: 2px solid transparent;
+    }
+    .loading-spinner::before {
+      inset: 0;
+      border-top-color: var(--accent);
+      animation: spin 1s linear infinite;
+    }
+    .loading-spinner::after {
+      inset: 8px;
+      border-top-color: var(--accent2);
+      animation: spin .7s linear infinite reverse;
+    }
     @keyframes spin { to { transform: rotate(360deg); } }
+    .loading-text {
+      font-family: 'Bricolage Grotesque', sans-serif;
+      font-size: 20px;
+      font-weight: 600;
+      color: var(--text);
+    }
+    .loading-sub {
+      font-family: 'JetBrains Mono', monospace;
+      font-size: 12px;
+      color: var(--text2);
+      max-width: 360px;
+      text-align: center;
+    }
+    .loading-bar-wrap {
+      width: 300px;
+      height: 3px;
+      background: var(--bg3);
+      border-radius: 2px;
+      overflow: hidden;
+    }
+    .loading-bar {
+      height: 100%;
+      width: 0%;
+      background: linear-gradient(90deg, var(--accent), var(--accent2));
+      border-radius: 2px;
+      transition: width .3s;
+    }
+    .loading-file {
+      font-size: 11px;
+      font-family: 'JetBrains Mono', monospace;
+      color: var(--text3);
+    }
+    /* ─── Error Toast ────────────────────────────────────── */
+    #toast {
+      position: fixed;
+      bottom: 24px;
+      left: 50%;
+      transform: translateX(-50%) translateY(80px);
+      background: rgba(245,85,119,.15);
+      border: 1px solid rgba(245,85,119,.4);
+      color: var(--red);
+      padding: 10px 20px;
+      border-radius: 10px;
+      font-size: 13px;
+      font-family: 'JetBrains Mono', monospace;
+      z-index: 500;
+      transition: transform .3s;
+      max-width: 500px;
+      text-align: center;
+    }
+    #toast.show { transform: translateX(-50%) translateY(0); }
+    /* ─── Footer ─────────────────────────────────────────── */
+    footer {
+      padding: 12px 32px;
+      border-top: 1px solid var(--border);
+      display: flex;
+      align-items: center;
+      justify-content: space-between;
+      font-size: 11px;
+      color: var(--text3);
+      font-family: 'JetBrains Mono', monospace;
+      background: rgba(6,11,20,.8);
+    }
+    footer a {
+      color: var(--text2);
+      text-decoration: none;
+      transition: color .15s;
+    }
+    footer a:hover { color: var(--accent); }
+    /* ─── Scrollbar ──────────────────────────────────────── */
+    ::-webkit-scrollbar { width: 6px; }
+    ::-webkit-scrollbar-track { background: transparent; }
+    ::-webkit-scrollbar-thumb { background: var(--border); border-radius: 3px; }
+    ::-webkit-scrollbar-thumb:hover { background: var(--border2); }
+    /* ─── Model color indicator ──────────────────────────── */
+    .model-indicator {
+      display: flex;
+      align-items: center;
+      gap: 6px;
+      font-size: 11px;
+      font-family: 'JetBrains Mono', monospace;
+      color: var(--text2);
+    }
+    .model-indicator-dot {
+      width: 8px;
+      height: 8px;
+      border-radius: 50%;
+    }
+    /* ─── Responsive ─────────────────────────────────────── */
+    @media (max-width: 900px) {
+      header { padding: 0 16px; }
+      .model-bar { padding: 12px 16px; }
+      main { grid-template-columns: 1fr; }
+      .input-panel { border-right: none; border-bottom: 1px solid var(--border); }
+      .stats-row { grid-template-columns: repeat(2, 1fr); }
+      .stat-card:nth-child(2) { border-right: none; }
+      footer { flex-direction: column; gap: 4px; text-align: center; }
+    }
+    /* ─── Animations ─────────────────────────────────────── */
+    @keyframes fadeIn {
+      from { opacity: 0; transform: translateY(6px); }
+      to   { opacity: 1; transform: translateY(0); }
+    }
+    .fade-in {
+      animation: fadeIn .25s ease forwards;
+    }
   </style>
 </head>
+<body>
+<!-- Background -->
+<div class="bg-gradient"></div>
+<div class="dot-grid"></div>
+<div id="app">
+  <!-- Header -->
+  <header>
+    <div class="logo">
+      <div class="logo-hex">T</div>
+      <span class="logo-name">TokenLens</span>
+      <span class="logo-tag">v1.0</span>
+    </div>
+    <div class="header-right">
+      <div class="header-badge">
+        <span class="dot"></span>
+        <span>runs in-browser · no server · no GPU</span>
       </div>
+    </div>
+  </header>
+  <!-- Model Selector Bar -->
+  <div class="model-bar">
+    <div class="model-bar-label">▸ select tokenizer</div>
+    <div class="model-tabs" id="model-tabs">
+      <!-- populated by JS -->
+    </div>
+    <div class="custom-model-row">
+      <label>HF model id:</label>
+      <input class="custom-input" id="custom-model-input" type="text"
+        placeholder="e.g. deepseek-ai/DeepSeek-V4-Pro  or  Xenova/gpt2" />
+      <button class="btn" id="custom-model-btn">Load ↗</button>
+    </div>
+  </div>
+  <!-- Main -->
+  <main>
+    <!-- Left: Input -->
+    <div class="input-panel">
+      <div class="panel-header">
+        <div class="panel-title">
+          <div class="panel-title-icon">✎</div>
+          Input Text
+        </div>
+        <div class="sample-btns">
+          <button class="sample-btn" data-sample="poetry">Poetry</button>
+          <button class="sample-btn" data-sample="code">Code</button>
+          <button class="sample-btn" data-sample="multilingual">Multi-lingual</button>
+          <button class="sample-btn" data-sample="numbers">Numbers</button>
+          <button class="sample-btn" data-sample="clear">Clear</button>
+        </div>
       </div>
+      <textarea id="input-area"
+        placeholder="Type or paste text here to see how the tokenizer splits it into tokens…
+&#10;&#10;Try some special characters, code snippets, emojis 🦊, or multi-lingual text (日本語, العربية) to see how different models handle them differently."></textarea>
+      <div class="char-counter"><span id="char-count">0</span> characters</div>
+    </div>
+    <!-- Right: Output -->
+    <div class="output-panel">
+      <!-- Stats -->
+      <div class="stats-row">
+        <div class="stat-card" id="sc-tokens">
+          <div class="stat-label">Tokens</div>
+          <div class="stat-value" id="stat-tokens">—</div>
+          <div class="stat-sub" id="stat-model-name">no model loaded</div>
+        </div>
+        <div class="stat-card" id="sc-chars">
+          <div class="stat-label">Characters</div>
+          <div class="stat-value" id="stat-chars">—</div>
+          <div class="stat-sub">total input</div>
+        </div>
+        <div class="stat-card" id="sc-words">
+          <div class="stat-label">Words</div>
+          <div class="stat-value" id="stat-words">—</div>
+          <div class="stat-sub">approx</div>
+        </div>
+        <div class="stat-card" id="sc-ratio">
+          <div class="stat-label">Chars / Token</div>
+          <div class="stat-value" id="stat-ratio">—</div>
+          <div class="stat-sub">efficiency</div>
+        </div>
       </div>
+      <!-- View toggle -->
+      <div class="view-toggle">
+        <div class="toggle-group">
+          <button class="toggle-btn active" data-view="text">Text View</button>
+          <button class="toggle-btn" data-view="ids">ID Grid</button>
+          <button class="toggle-btn" data-view="list">Token List</button>
+        </div>
+        <div class="model-indicator" id="model-indicator">
+          <div class="model-indicator-dot" id="model-dot" style="background:#3d5a80"></div>
+          <span id="model-indicator-label">no model</span>
         </div>
       </div>
+      <!-- Token display area -->
+      <div class="token-display" id="token-display">
+        <div class="placeholder-msg" id="placeholder">
+          <div class="placeholder-icon">⬡</div>
+          <p>Select a model above and type something<br>to see tokenization in action</p>
         </div>
       </div>
+    </div><!-- /output-panel -->
   </main>
+  <footer>
+    <span>TokenLens — Powered by <a href="https://github.com/xenova/transformers.js" target="_blank">Transformers.js</a> · Runs entirely in your browser</span>
+    <span>Hover tokens to see IDs · Add models via custom input above</span>
   </footer>
+</div><!-- /app -->
+<!-- Loading Overlay -->
+<div id="loading-overlay">
+  <div class="loading-spinner"></div>
+  <div class="loading-text" id="loading-title">Loading Tokenizer</div>
+  <div class="loading-sub" id="loading-sub">Downloading tokenizer files from Hugging Face Hub…<br>This may take a moment on first load. Files are cached in your browser.</div>
+  <div class="loading-bar-wrap">
+    <div class="loading-bar" id="loading-bar"></div>
+  </div>
+  <div class="loading-file" id="loading-file"></div>
+</div>
+<!-- Toast -->
+<div id="toast"></div>
+<!-- ─────────────────────────────────────────────────────────
+     TokenLens Script
+     ─────────────────────────────────────────────────────────
+     Architecture:
+       • Uses @xenova/transformers (Transformers.js v2) via CDN
+       • Tokenizer files downloaded from HF Hub and cached in IndexedDB
+       • Extends easily: add entries to MODELS registry
+       • Supports BPE, WordPiece, SentencePiece, Unigram tokenizers
+     ─────────────────────────────────────────────────────────── -->
+<script type="module">
+import { AutoTokenizer, env }
+  from 'https://cdn.jsdelivr.net/npm/@xenova/transformers@2.17.2';
+// ── Config ────────────────────────────────────────────────
+env.allowLocalModels = false;
+// Use HF CDN for model files
+env.useBrowserCache  = true;
+// ── Model Registry ─────────────────────────────────────────
+// Add any HuggingFace model ID here — tokenizer.json + tokenizer_config.json
+// are the only files downloaded (no weights, no GPU needed).
+const MODELS = [
+  {
+    id:    'Xenova/gpt2',
+    name:  'GPT-2',
+    org:   'OpenAI',
+    color: '#10a37f',
+    vocab: '50k',
+    type:  'BPE',
+    desc:  'Classic GPT-2 BPE tokenizer'
+  },
+  {
+    id:    'Xenova/gpt-4',
+    name:  'GPT-4',
+    org:   'OpenAI',
+    color: '#10a37f',
+    vocab: '100k',
+    type:  'tiktoken cl100k',
+    desc:  'Used by GPT-3.5 & GPT-4'
+  },
+  {
+    id:    'Xenova/llama-tokenizer',
+    name:  'LLaMA 2',
+    org:   'Meta',
+    color: '#0466de',
+    vocab: '32k',
+    type:  'SP-BPE',
+    desc:  'SentencePiece BPE — LLaMA / LLaMA-2'
+  },
+  {
+    id:    'Xenova/mistral-tokenizer-v1',
+    name:  'Mistral',
+    org:   'Mistral AI',
+    color: '#ff7722',
+    vocab: '32k',
+    type:  'SP-BPE',
+    desc:  'Mistral 7B v0.1 tokenizer'
+  },
+  {
+    id:    'Xenova/bert-base-uncased',
+    name:  'BERT',
+    org:   'Google',
+    color: '#4285f4',
+    vocab: '30k',
+    type:  'WordPiece',
+    desc:  'BERT-base uncased WordPiece'
+  },
+  {
+    id:    'Xenova/t5-base',
+    name:  'T5',
+    org:   'Google',
+    color: '#34a853',
+    vocab: '32k',
+    type:  'Unigram',
+    desc:  'T5 SentencePiece Unigram'
+  },
+  {
+    id:    'Xenova/claude-tokenizer',
+    name:  'Claude',
+    org:   'Anthropic',
+    color: '#cc785c',
+    vocab: '~100k',
+    type:  'BPE',
+    desc:  "Anthropic Claude's tokenizer"
+  },
+  {
+    id:    'Xenova/roberta-base',
+    name:  'RoBERTa',
+    org:   'Meta',
+    color: '#1a73e8',
+    vocab: '50k',
+    type:  'BPE',
+    desc:  'RoBERTa byte-level BPE'
+  },
+];
+// ── Token Color Palette ────────────────────────────────────
+const PALETTE = [
+  { text: '#ff8080', bg: 'rgba(255,128,128,.18)', border: 'rgba(255,128,128,.35)' },
+  { text: '#ffb84d', bg: 'rgba(255,184, 77,.18)', border: 'rgba(255,184, 77,.35)' },
+  { text: '#ffe066', bg: 'rgba(255,224,102,.18)', border: 'rgba(255,224,102,.35)' },
+  { text: '#7aed91', bg: 'rgba(122,237,145,.18)', border: 'rgba(122,237,145,.35)' },
+  { text: '#4ddfc0', bg: 'rgba( 77,223,192,.18)', border: 'rgba( 77,223,192,.35)' },
+  { text: '#56c8f5', bg: 'rgba( 86,200,245,.18)', border: 'rgba( 86,200,245,.35)' },
+  { text: '#748ef8', bg: 'rgba(116,142,248,.18)', border: 'rgba(116,142,248,.35)' },
+  { text: '#c484f8', bg: 'rgba(196,132,248,.18)', border: 'rgba(196,132,248,.35)' },
+  { text: '#f57cd4', bg: 'rgba(245,124,212,.18)', border: 'rgba(245,124,212,.35)' },
+  { text: '#fa8072', bg: 'rgba(250,128,114,.18)', border: 'rgba(250,128,114,.35)' },
+  { text: '#8be08b', bg: 'rgba(139,224,139,.18)', border: 'rgba(139,224,139,.35)' },
+  { text: '#f0c040', bg: 'rgba(240,192, 64,.18)', border: 'rgba(240,192, 64,.35)' },
+  { text: '#60d4e0', bg: 'rgba( 96,212,224,.18)', border: 'rgba( 96,212,224,.35)' },
+  { text: '#e89060', bg: 'rgba(232,144, 96,.18)', border: 'rgba(232,144, 96,.35)' },
+];
+// ── Sample texts ───────────────────────────────────────────
+const SAMPLES = {
+  poetry: `Two roads diverged in a yellow wood,
+And sorry I could not travel both
+And be one traveler, long I stood
+And looked down one as far as I could
+To where it bent in the undergrowth;
+— Robert Frost, "The Road Not Taken"`,
+  code: `async function fetchData(url, retries = 3) {
+  for (let i = 0; i < retries; i++) {
+    try {
+      const res = await fetch(url);
+      if (!res.ok) throw new Error(\`HTTP \${res.status}\`);
+      return await res.json();
+    } catch (e) {
+      if (i === retries - 1) throw e;
+      await new Promise(r => setTimeout(r, 1000 * 2 ** i));
     }
+  }
+}`,
+  multilingual: `English: The quick brown fox jumps over the lazy dog.
+日本語: 吾輩は猫である。名前はまだない。
+中文: 春眠不觉晓，处处闻啼鸟。
+العربية: اللغة العربية جميلة ومعبرة.
+Ελληνικά: Η γνώση είναι δύναμη.
+Emoji: 🌍 🦊 ⚡ 🎯 🧬 🤖 🦋`,
+  numbers: `π ≈ 3.14159265358979323846
+e ≈ 2.71828182845904523536
+φ ≈ 1.61803398874989484820
+1,000,000 × $42.99 = $42,990,000.00
+2024-01-15T08:30:00.000Z
+IPv4: 192.168.1.1 | IPv6: ::1`,
+  clear: ''
+};
+// ── State ──────────────────────────────────────────────────
+let activeTokenizer = null;
+let activeModel     = null;
+let tokenizerCache  = {};   // modelId → tokenizer
+let currentView     = 'text';
+let showSpecial     = false;
+let debounceTimer   = null;
+// ── DOM References ─────────────────────────────────────────
+const $overlay     = document.getElementById('loading-overlay');
+const $loadTitle   = document.getElementById('loading-title');
+const $loadSub     = document.getElementById('loading-sub');
+const $loadBar     = document.getElementById('loading-bar');
+const $loadFile    = document.getElementById('loading-file');
+const $modelTabs   = document.getElementById('model-tabs');
+const $input       = document.getElementById('input-area');
+const $charCount   = document.getElementById('char-count');
+const $display     = document.getElementById('token-display');
+const $placeholder = document.getElementById('placeholder');
+const $stTokens    = document.getElementById('stat-tokens');
+const $stChars     = document.getElementById('stat-chars');
+const $stWords     = document.getElementById('stat-words');
+const $stRatio     = document.getElementById('stat-ratio');
+const $stModelName = document.getElementById('stat-model-name');
+const $modelDot    = document.getElementById('model-dot');
+const $modelLabel  = document.getElementById('model-indicator-label');
+const $toast       = document.getElementById('toast');
+const $customInput = document.getElementById('custom-model-input');
+const $customBtn   = document.getElementById('custom-model-btn');
+// ── Utilities ──────────────────────────────────────────────
+function showOverlay(title, sub) {
+  $loadTitle.textContent = title;
+  $loadSub.textContent   = sub;
+  $loadBar.style.width   = '0%';
+  $loadFile.textContent  = '';
+  $overlay.classList.remove('hidden');
+}
+function hideOverlay() {
+  $overlay.classList.add('hidden');
+}
+function showToast(msg, duration = 5000) {
+  $toast.textContent = msg;
+  $toast.classList.add('show');
+  setTimeout(() => $toast.classList.remove('show'), duration);
+}
+function setStats(tokens, text) {
+  const chars = text.length;
+  const words = text.trim() ? text.trim().split(/\s+/).length : 0;
+  const ratio = tokens > 0 && chars > 0 ? (chars / tokens).toFixed(2) : '—';
+  $stTokens.textContent = tokens > 0 ? tokens.toLocaleString() : '—';
+  $stChars.textContent  = chars > 0  ? chars.toLocaleString()  : '—';
+  $stWords.textContent  = words > 0  ? words.toLocaleString()  : '—';
+  $stRatio.textContent  = ratio;
+  // Pulse animation
+  ['sc-tokens','sc-chars','sc-words','sc-ratio'].forEach(id => {
+    const el = document.getElementById(id);
+    el.classList.remove('highlight');
+    void el.offsetWidth;
+    el.classList.add('highlight');
+  });
+}
+// ── Decode raw token string for display ───���───────────────
+// Handles BPE Ġ prefix, SentencePiece ▁ prefix, byte tokens, etc.
+function decodeTokenString(raw) {
+  if (!raw) return '';
+  // BPE space prefix
+  let s = raw.replace(/^Ġ/, ' ').replace(/Ġ/g, ' ');
+  // SentencePiece space prefix
+  s = s.replace(/^▁/, ' ').replace(/▁/g, ' ');
+  // Newline representation
+  s = s.replace(/Ċ/g, '\n');
+  // Carriage return
+  s = s.replace(/\r/g, '');
+  // Byte tokens like <0xAB>
+  s = s.replace(/<0x([0-9A-Fa-f]{2})>/g, (_, hex) => {
+    const code = parseInt(hex, 16);
+    return code < 128 ? String.fromCharCode(code) : `[0x${hex}]`;
+  });
+  return s;
+}
+// ── Tokenize ───────────────────────────────────────────────
+async function tokenize(text) {
+  if (!activeTokenizer || !text.trim()) {
+    $display.innerHTML = '';
+    $display.appendChild($placeholder);
+    $placeholder.style.display = 'flex';
+    setStats(0, text);
+    return;
+  }
+  try {
+    $placeholder.style.display = 'none';
+    // Run tokenizer — only tokenize, no special tokens by default
+    const encoded = await activeTokenizer(text, {
+      add_special_tokens: showSpecial,
+      return_offsets_mapping: false,
     });
+    const ids = Array.from(encoded.input_ids.data);
+    // Get raw token strings
+    let rawTokens;
+    try {
+      rawTokens = activeTokenizer.model.convert_ids_to_tokens(ids);
+    } catch {
+      // Fallback: decode each token individually
+      rawTokens = await Promise.all(
+        ids.map(id => activeTokenizer.decode([id], { skip_special_tokens: false }))
+      );
+    }
+    // Pair: { id, raw, display }
+    const tokens = ids.map((id, i) => ({
+      id,
+      raw:     rawTokens[i] || '',
+      display: decodeTokenString(rawTokens[i] || ''),
+    }));
+    setStats(tokens.length, text);
+    renderView(tokens);
+  } catch (err) {
+    console.error('Tokenization error:', err);
+    showToast('Tokenization error: ' + err.message);
+  }
+}
+// ── Render Views ───────────────────────────────────────────
+function renderView(tokens) {
+  if (currentView === 'text') renderTextView(tokens);
+  else if (currentView === 'ids') renderIdView(tokens);
+  else if (currentView === 'list') renderListView(tokens);
+}
+function renderTextView(tokens) {
+  const container = document.createElement('div');
+  container.className = 'token-text-view fade-in';
+  tokens.forEach((tok, i) => {
+    const c = PALETTE[i % PALETTE.length];
+    const span = document.createElement('span');
+    span.className = 'tok';
+    span.style.background   = c.bg;
+    span.style.color         = c.text;
+    span.style.borderBottom  = `2px solid ${c.border}`;
+    // Display text — handle spaces and newlines visually
+    const disp = tok.display;
+    if (disp === ' ') {
+      span.innerHTML = '&nbsp;';
+    } else if (disp === '\n') {
+      span.innerHTML = '↵<br>';
+    } else if (disp === '\t') {
+      span.innerHTML = '→&nbsp;&nbsp;&nbsp;';
+    } else {
+      span.textContent = disp;
+    }
+    // Tooltip
+    const tip = document.createElement('div');
+    tip.className = 'tok-tooltip';
+    const rawEsc = tok.raw
+      .replace(/&/g,'&amp;')
+      .replace(/</g,'&lt;')
+      .replace(/>/g,'&gt;');
+    tip.innerHTML =
+      `<span class="tok-tooltip-id">#${tok.id}</span> · ` +
+      `<span class="tok-tooltip-text">${rawEsc || '(empty)'}</span>`;
+    span.appendChild(tip);
+    container.appendChild(span);
+  });
+  $display.innerHTML = '';
+  $display.appendChild(container);
+}
+function renderIdView(tokens) {
+  const container = document.createElement('div');
+  container.className = 'token-id-view fade-in';
+  tokens.forEach((tok, i) => {
+    const c = PALETTE[i % PALETTE.length];
+    const card = document.createElement('div');
+    card.className  = 'tok-id-card';
+    card.style.background   = c.bg;
+    card.style.borderColor  = c.border;
+    card.title = `Raw: ${tok.raw}`;
+    const top = document.createElement('div');
+    top.className = 'tok-id-top';
+    top.style.color = c.text;
+    top.textContent = tok.id;
+    const bot = document.createElement('div');
+    bot.className = 'tok-id-bottom';
+    // Show abbreviated display text
+    const label = tok.display.slice(0, 8).replace(/\n/g,'↵').replace(/\t/g,'→');
+    bot.textContent = label || '…';
+    card.appendChild(top);
+    card.appendChild(bot);
+    container.appendChild(card);
+  });
+  $display.innerHTML = '';
+  $display.appendChild(container);
+}
+function renderListView(tokens) {
+  const container = document.createElement('div');
+  container.className = 'token-split-view fade-in';
+  tokens.forEach((tok, i) => {
+    const c = PALETTE[i % PALETTE.length];
+    const row = document.createElement('div');
+    row.className = 'tok-split-row';
+    row.style.background  = c.bg;
+    row.style.borderColor = c.border;
+    const idx = document.createElement('div');
+    idx.className = 'tok-split-idx';
+    idx.textContent = i;
+    const text = document.createElement('div');
+    text.className = 'tok-split-text';
+    text.style.color = c.text;
+    const disp = tok.display.replace(/\n/g,'↵').replace(/\t/g,'→') || '(empty)';
+    text.textContent = disp;
+    const id = document.createElement('div');
+    id.className = 'tok-split-id';
+    id.textContent = tok.id;
+    row.appendChild(idx);
+    row.appendChild(text);
+    row.appendChild(id);
+    container.appendChild(row);
+  });
+  $display.innerHTML = '';
+  $display.appendChild(container);
+}
+// ── Load Tokenizer ─────────────────────────────────────────
+async function loadModel(modelId) {
+  if (tokenizerCache[modelId]) {
+    activeTokenizer = tokenizerCache[modelId];
+    updateModelIndicator(modelId);
+    await runTokenize();
+    return;
+  }
+  const displayName = modelId.split('/').pop();
+  showOverlay(
+    `Loading ${displayName}`,
+    `Fetching tokenizer.json and tokenizer_config.json from Hugging Face Hub.\nFiles are cached in IndexedDB after first download.`
+  );
+  let lastProgress = 0;
+  try {
+    const tokenizer = await AutoTokenizer.from_pretrained(modelId, {
+      progress_callback: (info) => {
+        if (info.status === 'downloading') {
+          const pct = info.total
+            ? Math.round((info.loaded / info.total) * 100)
+            : lastProgress;
+          $loadBar.style.width = pct + '%';
+          $loadFile.textContent = info.file || '';
+          lastProgress = pct;
+        } else if (info.status === 'done') {
+          $loadBar.style.width = '100%';
         }
       }
     });
+    tokenizerCache[modelId] = tokenizer;
+    activeTokenizer = tokenizer;
+    activeModel = modelId;
+    updateModelIndicator(modelId);
+    hideOverlay();
+    await runTokenize();
+  } catch (err) {
+    hideOverlay();
+    console.error('Failed to load tokenizer:', err);
+    showToast(`Failed to load "${modelId}": ${err.message}. Check the model ID and try again.`, 8000);
+  }
+}
+function updateModelIndicator(modelId) {
+  const preset = MODELS.find(m => m.id === modelId);
+  const color = preset ? preset.color : '#7899c0';
+  const name  = modelId.split('/').pop();
+  $modelDot.style.background = color;
+  $modelDot.style.boxShadow  = `0 0 6px ${color}`;
+  $modelLabel.textContent    = name;
+  $stModelName.textContent   = preset ? `${preset.org} · ${preset.type} · ${preset.vocab} vocab` : modelId;
+}
+// ── Build Model Tabs ───────────────────────────────────────
+function buildTabs() {
+  $modelTabs.innerHTML = '';
+  MODELS.forEach(m => {
+    const tab = document.createElement('div');
+    tab.className   = 'model-tab';
+    tab.dataset.id  = m.id;
+    tab.title       = m.desc;
+    tab.innerHTML = `
+      <div class="model-tab-name">${m.name}</div>
+      <div class="model-tab-org">
+        <span class="model-org-dot" style="background:${m.color}"></span>${m.org}
+      </div>
+      <div class="model-tab-vocab">${m.type} · ${m.vocab} vocab</div>
+    `;
+    tab.addEventListener('click', () => selectTab(m.id));
+    $modelTabs.appendChild(tab);
+  });
+}
+function selectTab(modelId) {
+  document.querySelectorAll('.model-tab').forEach(t => {
+    t.classList.toggle('active', t.dataset.id === modelId);
+  });
+  loadModel(modelId);
+}
+// ── View Toggle ────────────────────────────────────────────
+document.querySelectorAll('.toggle-btn').forEach(btn => {
+  btn.addEventListener('click', () => {
+    document.querySelectorAll('.toggle-btn').forEach(b => b.classList.remove('active'));
+    btn.classList.add('active');
+    currentView = btn.dataset.view;
+    runTokenize();
+  });
+});
+// ── Input Handling ─────────────────────────────────────────
+async function runTokenize() {
+  const text = $input.value;
+  $charCount.textContent = text.length;
+  await tokenize(text);
+}
+$input.addEventListener('input', () => {
+  $charCount.textContent = $input.value.length;
+  clearTimeout(debounceTimer);
+  debounceTimer = setTimeout(runTokenize, 280);
+});
+// ── Sample Buttons ─────────────────────────────────────────
+document.querySelectorAll('.sample-btn').forEach(btn => {
+  btn.addEventListener('click', () => {
+    const key = btn.dataset.sample;
+    $input.value = SAMPLES[key] ?? '';
+    $input.focus();
+    runTokenize();
+  });
+});
+// ── Custom Model ───────────────────────────────────────────
+async function loadCustomModel() {
+  const id = $customInput.value.trim();
+  if (!id) { showToast('Please enter a model ID'); return; }
+  // Deselect tabs
+  document.querySelectorAll('.model-tab').forEach(t => t.classList.remove('active'));
+  activeModel = id;
+  await loadModel(id);
+}
+$customBtn.addEventListener('click', loadCustomModel);
+$customInput.addEventListener('keydown', e => {
+  if (e.key === 'Enter') loadCustomModel();
+});
+// ── Init ───────────────────────────────────────────────────
+buildTabs();
+// Hide overlay on start (no model yet)
+$overlay.classList.add('hidden');
+// Default placeholder text
+$input.value = '';
+// Auto-select first model
+selectTab(MODELS[0].id);
+</script>
+</body>
+</html>