v.1.1

index.html

@@ -108,6 +108,7 @@
       position: sticky;
       top: 0;
       z-index: 100;
+      gap: 16px;
     }
 
     .logo {
@@ -116,6 +117,7 @@
       gap: 10px;
       text-decoration: none;
       color: var(--text);
+      flex-shrink: 0;
     }
     .logo-hex {
       width: 34px;
@@ -151,65 +153,25 @@
       letter-spacing: .5px;
     }
 
-    .header-right {
-      display: flex;
-      align-items: center;
-      gap: 16px;
-    }
-    .header-badge {
-      display: flex;
-      align-items: center;
-      gap: 6px;
-      font-size: 12px;
-      color: var(--text2);
-      font-family: 'JetBrains Mono', monospace;
-    }
-    .header-badge .dot {
-      width: 7px;
-      height: 7px;
-      border-radius: 50%;
-      background: var(--green);
-      box-shadow: 0 0 8px var(--green);
-      animation: pulse-dot 2s ease-in-out infinite;
-    }
-    @keyframes pulse-dot {
-      0%,100% { opacity: 1; }
-      50% { opacity: .4; }
-    }
-
-    /* ─── Model Selector Bar ─────────────────────────────── */
-    .model-bar {
-      padding: 16px 32px;
-      border-bottom: 1px solid var(--border);
-      background: rgba(11,18,32,.7);
-      backdrop-filter: blur(12px);
-    }
-    .model-bar-label {
-      font-size: 11px;
-      font-family: 'JetBrains Mono', monospace;
-      color: var(--text3);
-      letter-spacing: 1.5px;
-      text-transform: uppercase;
-      margin-bottom: 10px;
-    }
+    /* Model tabs in header */
     .model-tabs {
       display: flex;
-      flex-wrap: wrap;
-      gap: 8px;
+      gap: 6px;
       align-items: center;
+      flex-shrink: 0;
     }
     .model-tab {
       display: flex;
       flex-direction: column;
-      padding: 8px 14px;
+      padding: 6px 10px;
       border: 1px solid var(--border);
-      border-radius: 10px;
+      border-radius: 8px;
       background: var(--bg2);
       cursor: pointer;
       transition: all 0.2s ease;
       position: relative;
       overflow: hidden;
-      min-width: 110px;
+      min-width: 90px;
     }
     .model-tab::before {
       content: '';
@@ -258,12 +220,12 @@
       top: -1px;
     }
 
-    /* Custom model row */
+    /* Custom model row in header */
     .custom-model-row {
       display: flex;
       align-items: center;
-      gap: 10px;
-      margin-top: 12px;
+      gap: 8px;
+      flex-shrink: 0;
     }
     .custom-model-row label {
       font-size: 11px;
@@ -272,22 +234,22 @@
       white-space: nowrap;
     }
     .custom-input {
-      flex: 1;
-      max-width: 380px;
+      width: 180px;
+      max-width: 220px;
       background: var(--bg2);
       border: 1px solid var(--border);
       border-radius: 8px;
       color: var(--text);
       font-family: 'JetBrains Mono', monospace;
       font-size: 13px;
-      padding: 7px 12px;
+      padding: 6px 10px;
       outline: none;
       transition: border-color 0.2s;
     }
     .custom-input:focus { border-color: var(--accent); }
     .custom-input::placeholder { color: var(--text3); }
     .btn {
-      padding: 7px 16px;
+      padding: 6px 14px;
       border-radius: 8px;
       border: 1px solid var(--border2);
       background: linear-gradient(135deg, rgba(77,158,245,.15), rgba(139,106,245,.15));
@@ -802,7 +764,6 @@
     /* ─── Responsive ─────────────────────────────────────── */
     @media (max-width: 900px) {
       header { padding: 0 16px; }
-      .model-bar { padding: 12px 16px; }
       main { grid-template-columns: 1fr; }
       .input-panel { border-right: none; border-bottom: 1px solid var(--border); }
       .stats-row { grid-template-columns: repeat(2, 1fr); }
@@ -833,29 +794,20 @@
     <div class="logo">
       <div class="logo-hex">T</div>
       <span class="logo-name">TokenLens</span>
-      <span class="logo-tag">v1.0</span>
+      <span class="logo-tag">v1.1</span>
     </div>
-    <div class="header-right">
-      <div class="header-badge">
-        <span class="dot"></span>
-        <span>runs in-browser · no server · no GPU</span>
-      </div>
-    </div>
-  </header>
-
-  <!-- Model Selector Bar -->
-  <div class="model-bar">
-    <div class="model-bar-label">▸ select tokenizer</div>
+    
     <div class="model-tabs" id="model-tabs">
       <!-- populated by JS -->
     </div>
+    
     <div class="custom-model-row">
       <label>HF model id:</label>
       <input class="custom-input" id="custom-model-input" type="text"
-        placeholder="e.g. deepseek-ai/DeepSeek-V4-Pro  or  Xenova/gpt2" />
+        placeholder="e.g. Xenova/gpt2" />
       <button class="btn" id="custom-model-btn">Load ↗</button>
     </div>
-  </div>
+  </header>
 
   <!-- Main -->
   <main>
@@ -973,18 +925,7 @@ env.allowLocalModels = false;
 env.useBrowserCache  = true;
 
 // ── Model Registry ─────────────────────────────────────────
-// Add any HuggingFace model ID here — tokenizer.json + tokenizer_config.json
-// are the only files downloaded (no weights, no GPU needed).
 const MODELS = [
-  {
-    id:    'Xenova/gpt2',
-    name:  'GPT-2',
-    org:   'OpenAI',
-    color: '#10a37f',
-    vocab: '50k',
-    type:  'BPE',
-    desc:  'Classic GPT-2 BPE tokenizer'
-  },
   {
     id:    'Xenova/gpt-4',
     name:  'GPT-4',
@@ -994,15 +935,6 @@ const MODELS = [
     type:  'tiktoken cl100k',
     desc:  'Used by GPT-3.5 & GPT-4'
   },
-  {
-    id:    'Xenova/llama-tokenizer',
-    name:  'LLaMA 2',
-    org:   'Meta',
-    color: '#0466de',
-    vocab: '32k',
-    type:  'SP-BPE',
-    desc:  'SentencePiece BPE — LLaMA / LLaMA-2'
-  },
   {
     id:    'Xenova/mistral-tokenizer-v1',
     name:  'Mistral',
@@ -1012,24 +944,6 @@ const MODELS = [
     type:  'SP-BPE',
     desc:  'Mistral 7B v0.1 tokenizer'
   },
-  {
-    id:    'Xenova/bert-base-uncased',
-    name:  'BERT',
-    org:   'Google',
-    color: '#4285f4',
-    vocab: '30k',
-    type:  'WordPiece',
-    desc:  'BERT-base uncased WordPiece'
-  },
-  {
-    id:    'Xenova/t5-base',
-    name:  'T5',
-    org:   'Google',
-    color: '#34a853',
-    vocab: '32k',
-    type:  'Unigram',
-    desc:  'T5 SentencePiece Unigram'
-  },
   {
     id:    'Xenova/claude-tokenizer',
     name:  'Claude',
@@ -1039,15 +953,6 @@ const MODELS = [
     type:  'BPE',
     desc:  "Anthropic Claude's tokenizer"
   },
-  {
-    id:    'Xenova/roberta-base',
-    name:  'RoBERTa',
-    org:   'Meta',
-    color: '#1a73e8',
-    vocab: '50k',
-    type:  'BPE',
-    desc:  'RoBERTa byte-level BPE'
-  },
 ];
 
 // ── Token Color Palette ────────────────────────────────────
@@ -1178,18 +1083,12 @@ function setStats(tokens, text) {
 }
 
 // ── Decode raw token string for display ───────────────────
-// Handles BPE Ġ prefix, SentencePiece ▁ prefix, byte tokens, etc.
 function decodeTokenString(raw) {
   if (!raw) return '';
-  // BPE space prefix
   let s = raw.replace(/^Ġ/, ' ').replace(/Ġ/g, ' ');
-  // SentencePiece space prefix
   s = s.replace(/^▁/, ' ').replace(/▁/g, ' ');
-  // Newline representation
   s = s.replace(/Ċ/g, '\n');
-  // Carriage return
   s = s.replace(/\r/g, '');
-  // Byte tokens like <0xAB>
   s = s.replace(/<0x([0-9A-Fa-f]{2})>/g, (_, hex) => {
     const code = parseInt(hex, 16);
     return code < 128 ? String.fromCharCode(code) : `[0x${hex}]`;
@@ -1210,7 +1109,6 @@ async function tokenize(text) {
   try {
     $placeholder.style.display = 'none';
 
-    // Run tokenizer — only tokenize, no special tokens by default
     const encoded = await activeTokenizer(text, {
       add_special_tokens: showSpecial,
       return_offsets_mapping: false,
@@ -1218,18 +1116,15 @@ async function tokenize(text) {
 
     const ids = Array.from(encoded.input_ids.data);
 
-    // Get raw token strings
     let rawTokens;
     try {
       rawTokens = activeTokenizer.model.convert_ids_to_tokens(ids);
     } catch {
-      // Fallback: decode each token individually
       rawTokens = await Promise.all(
         ids.map(id => activeTokenizer.decode([id], { skip_special_tokens: false }))
       );
     }
 
-    // Pair: { id, raw, display }
     const tokens = ids.map((id, i) => ({
       id,
       raw:     rawTokens[i] || '',
@@ -1265,7 +1160,6 @@ function renderTextView(tokens) {
     span.style.color         = c.text;
     span.style.borderBottom  = `2px solid ${c.border}`;
 
-    // Display text — handle spaces and newlines visually
     const disp = tok.display;
     if (disp === ' ') {
       span.innerHTML = '&nbsp;';
@@ -1277,7 +1171,6 @@ function renderTextView(tokens) {
       span.textContent = disp;
     }
 
-    // Tooltip
     const tip = document.createElement('div');
     tip.className = 'tok-tooltip';
 
@@ -1317,7 +1210,6 @@ function renderIdView(tokens) {
 
     const bot = document.createElement('div');
     bot.className = 'tok-id-bottom';
-    // Show abbreviated display text
     const label = tok.display.slice(0, 8).replace(/\n/g,'↵').replace(/\t/g,'→');
     bot.textContent = label || '…';
 
@@ -1494,7 +1386,6 @@ async function loadCustomModel() {
   const id = $customInput.value.trim();
   if (!id) { showToast('Please enter a model ID'); return; }
 
-  // Deselect tabs
   document.querySelectorAll('.model-tab').forEach(t => t.classList.remove('active'));
   activeModel = id;
   await loadModel(id);
@@ -1508,15 +1399,12 @@ $customInput.addEventListener('keydown', e => {
 // ── Init ───────────────────────────────────────────────────
 
 buildTabs();
-// Hide overlay on start (no model yet)
 $overlay.classList.add('hidden');
 
-// Default placeholder text
 $input.value = '';
 
-// Auto-select first model
 selectTab(MODELS[0].id);
 
 </script>
 </body>
-</html>
+</html>