v1.4

index.html

@@ -576,7 +576,7 @@
     <div class="logo">
       <div class="logo-hex">T</div>
       <span class="logo-name">TokenLens</span>
-      <span class="logo-tag">v1.3</span>
+      <span class="logo-tag">v1.4</span>
     </div>
     <div class="header-divider"></div>
     <div class="header-controls">
@@ -747,21 +747,76 @@ Try special characters, code, emojis, or multi-lingual text to compare models.">
 
 <script type="module">
 import { AutoTokenizer, env }
-  from 'https://cdn.jsdelivr.net/npm/@xenova/transformers@2.17.2';
+  from 'https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0';
 
 env.allowLocalModels = false;
 env.useBrowserCache  = true;
 
 // ── Model Registry ─────────────────────────────────────────
 const MODELS = [
-  { id:'Xenova/gpt2', name:'GPT-2', org:'OpenAI', color:'#10a37f', vocab:'50k', type:'BPE', desc:'Classic GPT-2 BPE tokenizer' },
-  { id:'Xenova/gpt-4', name:'GPT-4', org:'OpenAI', color:'#10a37f', vocab:'100k', type:'tiktoken cl100k', desc:'Used by GPT-3.5 & GPT-4' },
-  { id:'Xenova/llama-tokenizer', name:'LLaMA 2', org:'Meta', color:'#0466de', vocab:'32k', type:'SP-BPE', desc:'SentencePiece BPE — LLaMA / LLaMA-2' },
-  { id:'Xenova/mistral-tokenizer-v1', name:'Mistral', org:'Mistral AI', color:'#ff7722', vocab:'32k', type:'SP-BPE', desc:'Mistral 7B v0.1 tokenizer' },
-  { id:'Xenova/bert-base-uncased', name:'BERT', org:'Google', color:'#4285f4', vocab:'30k', type:'WordPiece', desc:'BERT-base uncased WordPiece' },
-  { id:'Xenova/t5-base', name:'T5', org:'Google', color:'#34a853', vocab:'32k', type:'Unigram', desc:'T5 SentencePiece Unigram' },
-  { id:'Xenova/claude-tokenizer', name:'Claude', org:'Anthropic', color:'#cc785c', vocab:'~100k', type:'BPE', desc:"Anthropic Claude's tokenizer" },
-  { id:'Xenova/roberta-base', name:'RoBERTa', org:'Meta', color:'#1a73e8', vocab:'50k', type:'BPE', desc:'RoBERTa byte-level BPE' },
+  {
+    id:'Qwen/Qwen3.6-27B',
+    name:'Qwen 3.6 27B',
+    org:'Alibaba',
+    color:'#0466de',
+    vocab:'152k',
+    type:'BPE',
+    desc:'Qwen tokenizer optimized for multilingual coding and reasoning tasks'
+  },
+  {
+    id:'deepseek-ai/DeepSeek-V4-Pro',
+    name:'DeepSeek V4 Pro',
+    org:'DeepSeek',
+    color:'#4285f4',
+    vocab:'129k',
+    type:'BPE',
+    desc:'DeepSeek multilingual tokenizer designed for code-heavy workloads'
+  },
+  {
+    id:'MiniMaxAI/MiniMax-M2.7',
+    name:'MiniMax M2.7',
+    org:'MiniMax',
+    color:'#1a73e8',
+    vocab:'128k',
+    type:'SentencePiece',
+    desc:'Efficient multilingual tokenizer for long-context multimodal models'
+  },
+  {
+    id:'mistralai/Mistral-Medium-3.5-128B',
+    name:'Mistral Medium 3.5',
+    org:'Mistral AI',
+    color:'#ff7722',
+    vocab:'131k',
+    type:'Tekken BPE',
+    desc:'Mistral Tekken tokenizer with efficient multilingual compression'
+  },
+  {
+    id:'google/gemma-4-31B-it',
+    name:'Gemma 4 31B',
+    org:'Google',
+    color:'#34a853',
+    vocab:'256k',
+    type:'SentencePiece',
+    desc:'Gemma multilingual SentencePiece tokenizer with large vocabulary'
+  },  
+  {
+    id:'zai-org/GLM-5.1',
+    name:'GLM-5.1',
+    org:'Z.ai',
+    color:'#10a37f',
+    vocab:'151k',
+    type:'SentencePiece',
+    desc:'GLM multilingual SentencePiece tokenizer with large bilingual vocabulary'
+  },
+  {
+    id:'XiaomiMiMo/MiMo-V2.5-Pro',
+    name:'MiMo V2.5 Pro',
+    org:'Xiaomi',
+    color:'#ff7722',
+    vocab:'128k',
+    type:'SentencePiece',
+    desc:'MoE tokenizer tuned for multilingual reasoning and code generation'
+  },
 ];
 
 // ── Token Color Palettes ───────────────────────────────────