Spaces:
Running
Running
Update index.html
Browse files- index.html +293 -456
index.html
CHANGED
|
@@ -1,465 +1,302 @@
|
|
| 1 |
<!DOCTYPE html>
|
| 2 |
-
<html lang="en"
|
| 3 |
<head>
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
}
|
| 34 |
-
.token-box {
|
| 35 |
-
white-space: pre;
|
| 36 |
-
word-break: break-all;
|
| 37 |
-
}
|
| 38 |
-
}
|
| 39 |
-
</style>
|
| 40 |
</head>
|
| 41 |
-
<body class="
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
disabled
|
| 173 |
-
>
|
| 174 |
-
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
| 175 |
-
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M8 16H6a2 2 0 01-2-2V6a2 2 0 012-2h8a2 2 0 012 2v2m-6 12h8a2 2 0 002-2v-8a2 2 0 00-2-2h-8a2 2 0 00-2 2v8a2 2 0 002 2z"></path>
|
| 176 |
-
</svg>
|
| 177 |
-
Copy IDs
|
| 178 |
-
</button>
|
| 179 |
-
</div>
|
| 180 |
-
</div>
|
| 181 |
-
<div
|
| 182 |
-
id="tokenContainer"
|
| 183 |
-
class="min-h-[120px] p-4 bg-gray-50 dark:bg-gray-700/50 rounded-lg overflow-x-auto flex flex-wrap gap-2 items-start content-start"
|
| 184 |
-
>
|
| 185 |
-
<p id="emptyState" class="text-gray-500 dark:text-gray-400 w-full text-center py-8">Load a model and enter text to see tokens</p>
|
| 186 |
-
</div>
|
| 187 |
-
<p class="text-xs text-gray-500 dark:text-gray-400 mt-2">Hover over tokens to see their ID. Special tokens are marked with a dashed border.</p>
|
| 188 |
-
</section>
|
| 189 |
-
|
| 190 |
-
<!-- Footer -->
|
| 191 |
-
<footer class="text-center text-gray-600 dark:text-gray-400 text-sm">
|
| 192 |
-
<p>Powered by <a href="https://huggingface.co/docs/transformers.js" target="_blank" class="text-blue-600 dark:text-blue-400 hover:underline">Transformers.js</a> | Runs entirely in your browser, no server required</p>
|
| 193 |
-
</footer>
|
| 194 |
-
</div>
|
| 195 |
-
|
| 196 |
-
<script type="module">
|
| 197 |
-
// Wait for Transformers.js to load
|
| 198 |
-
await new Promise(resolve => {
|
| 199 |
-
const checkInterval = setInterval(() => {
|
| 200 |
-
if (window.AutoTokenizer) clearInterval(checkInterval);
|
| 201 |
-
}, 100);
|
| 202 |
-
});
|
| 203 |
-
|
| 204 |
-
const { AutoTokenizer } = window;
|
| 205 |
-
|
| 206 |
-
// DOM Elements
|
| 207 |
-
const themeToggle = document.getElementById('themeToggle');
|
| 208 |
-
const sunIcon = document.getElementById('sunIcon');
|
| 209 |
-
const moonIcon = document.getElementById('moonIcon');
|
| 210 |
-
const modelIdInput = document.getElementById('modelId');
|
| 211 |
-
const hfTokenInput = document.getElementById('hfToken');
|
| 212 |
-
const loadModelBtn = document.getElementById('loadModelBtn');
|
| 213 |
-
const loadSpinner = document.getElementById('loadSpinner');
|
| 214 |
-
const loadBtnText = document.getElementById('loadBtnText');
|
| 215 |
-
const popularModelsSelect = document.getElementById('popularModels');
|
| 216 |
-
const modelError = document.getElementById('modelError');
|
| 217 |
-
const modelSuccess = document.getElementById('modelSuccess');
|
| 218 |
-
const textInput = document.getElementById('textInput');
|
| 219 |
-
const clearTextBtn = document.getElementById('clearTextBtn');
|
| 220 |
-
const tokenContainer = document.getElementById('tokenContainer');
|
| 221 |
-
const emptyState = document.getElementById('emptyState');
|
| 222 |
-
const totalTokensEl = document.getElementById('totalTokens');
|
| 223 |
-
const totalCharsEl = document.getElementById('totalChars');
|
| 224 |
-
const tokenRatioEl = document.getElementById('tokenRatio');
|
| 225 |
-
const copyTokensBtn = document.getElementById('copyTokensBtn');
|
| 226 |
-
const copyIdsBtn = document.getElementById('copyIdsBtn');
|
| 227 |
-
|
| 228 |
-
// State
|
| 229 |
-
let tokenizer = null;
|
| 230 |
-
let currentTokens = [];
|
| 231 |
-
let currentTokenIds = [];
|
| 232 |
-
let isDark = localStorage.getItem('theme') === 'dark' || (!localStorage.getItem('theme') && window.matchMedia('(prefers-color-scheme: dark)').matches);
|
| 233 |
-
|
| 234 |
-
// Initialize theme
|
| 235 |
-
function initTheme() {
|
| 236 |
-
if (isDark) {
|
| 237 |
-
document.documentElement.classList.add('dark');
|
| 238 |
-
sunIcon.classList.remove('hidden');
|
| 239 |
-
moonIcon.classList.add('hidden');
|
| 240 |
-
} else {
|
| 241 |
-
document.documentElement.classList.remove('dark');
|
| 242 |
-
sunIcon.classList.add('hidden');
|
| 243 |
-
moonIcon.classList.remove('hidden');
|
| 244 |
-
}
|
| 245 |
-
}
|
| 246 |
-
initTheme();
|
| 247 |
-
|
| 248 |
-
// Theme toggle
|
| 249 |
-
themeToggle.addEventListener('click', () => {
|
| 250 |
-
isDark = !isDark;
|
| 251 |
-
localStorage.setItem('theme', isDark ? 'dark' : 'light');
|
| 252 |
-
initTheme();
|
| 253 |
-
// Re-render tokens to update colors
|
| 254 |
-
if (currentTokens.length > 0) renderTokens();
|
| 255 |
-
});
|
| 256 |
-
|
| 257 |
-
// Popular models select
|
| 258 |
-
popularModelsSelect.addEventListener('change', (e) => {
|
| 259 |
-
if (e.target.value) {
|
| 260 |
-
modelIdInput.value = e.target.value;
|
| 261 |
-
}
|
| 262 |
-
});
|
| 263 |
-
|
| 264 |
-
// Load model function
|
| 265 |
-
async function loadModel() {
|
| 266 |
-
const modelId = modelIdInput.value.trim();
|
| 267 |
-
if (!modelId) {
|
| 268 |
-
showError('Please enter a Hugging Face model ID');
|
| 269 |
-
return;
|
| 270 |
-
}
|
| 271 |
-
|
| 272 |
-
// Show loading state
|
| 273 |
-
loadModelBtn.disabled = true;
|
| 274 |
-
loadSpinner.classList.remove('hidden');
|
| 275 |
-
loadBtnText.textContent = 'Loading...';
|
| 276 |
-
hideMessages();
|
| 277 |
-
textInput.disabled = true;
|
| 278 |
-
|
| 279 |
-
try {
|
| 280 |
-
const token = hfTokenInput.value.trim() || undefined;
|
| 281 |
-
tokenizer = await AutoTokenizer.from_pretrained(modelId, { token });
|
| 282 |
-
|
| 283 |
-
// Success
|
| 284 |
-
showSuccess(`Successfully loaded tokenizer for ${modelId}`);
|
| 285 |
-
textInput.disabled = false;
|
| 286 |
-
textInput.focus();
|
| 287 |
-
|
| 288 |
-
// Auto-tokenize existing text
|
| 289 |
-
if (textInput.value.trim()) {
|
| 290 |
-
tokenizeText();
|
| 291 |
-
}
|
| 292 |
-
} catch (err) {
|
| 293 |
-
console.error('Failed to load model:', err);
|
| 294 |
-
showError(`Failed to load model: ${err.message}. Check the model ID and ensure it's public (or provide a HF token for gated models).`);
|
| 295 |
-
tokenizer = null;
|
| 296 |
-
textInput.disabled = true;
|
| 297 |
-
} finally {
|
| 298 |
-
// Reset button state
|
| 299 |
-
loadModelBtn.disabled = false;
|
| 300 |
-
loadSpinner.classList.add('hidden');
|
| 301 |
-
loadBtnText.textContent = 'Load Model';
|
| 302 |
}
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
// Tokenize text function
|
| 306 |
-
function tokenizeText() {
|
| 307 |
-
if (!tokenizer) return;
|
| 308 |
-
const text = textInput.value;
|
| 309 |
-
|
| 310 |
-
if (!text.trim()) {
|
| 311 |
-
currentTokens = [];
|
| 312 |
-
currentTokenIds = [];
|
| 313 |
-
renderTokens();
|
| 314 |
-
updateStats();
|
| 315 |
-
copyTokensBtn.disabled = true;
|
| 316 |
-
copyIdsBtn.disabled = true;
|
| 317 |
-
return;
|
| 318 |
-
}
|
| 319 |
-
|
| 320 |
-
try {
|
| 321 |
-
const output = tokenizer(text);
|
| 322 |
-
currentTokens = output.tokens;
|
| 323 |
-
currentTokenIds = output.input_ids;
|
| 324 |
-
renderTokens();
|
| 325 |
-
updateStats();
|
| 326 |
-
copyTokensBtn.disabled = false;
|
| 327 |
-
copyIdsBtn.disabled = false;
|
| 328 |
-
} catch (err) {
|
| 329 |
-
console.error('Tokenization failed:', err);
|
| 330 |
-
showError(`Tokenization failed: ${err.message}`);
|
| 331 |
-
}
|
| 332 |
-
}
|
| 333 |
-
|
| 334 |
-
// Render tokens
|
| 335 |
-
function renderTokens() {
|
| 336 |
-
tokenContainer.innerHTML = '';
|
| 337 |
-
|
| 338 |
-
if (currentTokens.length === 0) {
|
| 339 |
-
tokenContainer.appendChild(emptyState);
|
| 340 |
-
emptyState.classList.remove('hidden');
|
| 341 |
-
return;
|
| 342 |
-
}
|
| 343 |
-
|
| 344 |
-
emptyState.classList.add('hidden');
|
| 345 |
-
|
| 346 |
-
currentTokens.forEach((token, index) => {
|
| 347 |
-
const tokenId = currentTokenIds[index];
|
| 348 |
-
// Generate consistent pastel color based on token ID
|
| 349 |
-
const hue = (tokenId * 137) % 360; // 137 is prime for even distribution
|
| 350 |
-
const lightness = isDark ? '80%' : '90%';
|
| 351 |
-
const bgColor = `hsl(${hue}, 70%, ${lightness})`;
|
| 352 |
-
const textColor = isDark ? '#1f2937' : '#1f2937';
|
| 353 |
-
|
| 354 |
-
const tokenEl = document.createElement('span');
|
| 355 |
-
tokenEl.className = 'token-box px-2 py-1 rounded-md text-sm font-mono cursor-pointer transition-all hover:scale-105 hover:shadow-md';
|
| 356 |
-
tokenEl.style.backgroundColor = bgColor;
|
| 357 |
-
tokenEl.style.color = textColor;
|
| 358 |
-
|
| 359 |
-
// Replace whitespace with visible symbols
|
| 360 |
-
const displayToken = token
|
| 361 |
-
.replace(/\n/g, '↵')
|
| 362 |
-
.replace(/\t/g, '→')
|
| 363 |
-
.replace(/ /g, '·');
|
| 364 |
-
tokenEl.textContent = displayToken;
|
| 365 |
-
|
| 366 |
-
// Show token ID on hover
|
| 367 |
-
tokenEl.title = `Token ID: ${tokenId}\nRaw: ${token.replace(/\n/g, '\\n').replace(/\t/g, '\\t')}`;
|
| 368 |
-
|
| 369 |
-
// Highlight special tokens
|
| 370 |
-
if (tokenizer.special_tokens_map && Object.values(tokenizer.special_tokens_map).flat().includes(token)) {
|
| 371 |
-
tokenEl.classList.add('border', 'border-dashed', 'border-gray-400', 'dark:border-gray-500');
|
| 372 |
-
}
|
| 373 |
-
|
| 374 |
-
tokenContainer.appendChild(tokenEl);
|
| 375 |
-
});
|
| 376 |
-
}
|
| 377 |
-
|
| 378 |
-
// Update stats
|
| 379 |
-
function updateStats() {
|
| 380 |
-
const totalTokens = currentTokens.length;
|
| 381 |
-
const totalChars = textInput.value.length;
|
| 382 |
-
const ratio = totalChars > 0 ? (totalTokens / totalChars).toFixed(2) : '0.00';
|
| 383 |
-
|
| 384 |
-
totalTokensEl.textContent = totalTokens.toLocaleString();
|
| 385 |
-
totalCharsEl.textContent = totalChars.toLocaleString();
|
| 386 |
-
tokenRatioEl.textContent = ratio;
|
| 387 |
-
}
|
| 388 |
-
|
| 389 |
-
// Copy functions
|
| 390 |
-
async function copyToClipboard(text, btn) {
|
| 391 |
-
try {
|
| 392 |
-
await navigator.clipboard.writeText(text);
|
| 393 |
-
const originalText = btn.innerHTML;
|
| 394 |
-
btn.innerHTML = `
|
| 395 |
-
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
| 396 |
-
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M5 13l4 4L19 7"></path>
|
| 397 |
-
</svg>
|
| 398 |
-
Copied!
|
| 399 |
-
`;
|
| 400 |
-
setTimeout(() => {
|
| 401 |
-
btn.innerHTML = originalText;
|
| 402 |
-
}, 2000);
|
| 403 |
-
} catch (err) {
|
| 404 |
-
showError('Failed to copy to clipboard');
|
| 405 |
-
}
|
| 406 |
-
}
|
| 407 |
-
|
| 408 |
-
copyTokensBtn.addEventListener('click', () => {
|
| 409 |
-
copyToClipboard(currentTokens.join('\n'), copyTokensBtn);
|
| 410 |
});
|
| 411 |
-
|
| 412 |
-
|
| 413 |
-
|
| 414 |
-
|
| 415 |
-
|
| 416 |
-
|
| 417 |
-
|
| 418 |
-
|
| 419 |
-
|
| 420 |
-
|
| 421 |
-
|
| 422 |
-
|
| 423 |
-
|
| 424 |
-
|
| 425 |
-
|
| 426 |
-
|
| 427 |
-
|
| 428 |
-
|
| 429 |
-
|
| 430 |
-
|
| 431 |
-
|
| 432 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 433 |
}
|
| 434 |
-
|
| 435 |
-
|
| 436 |
-
|
| 437 |
-
|
| 438 |
-
}
|
| 439 |
-
|
| 440 |
-
|
| 441 |
-
|
| 442 |
-
|
| 443 |
-
|
| 444 |
-
|
| 445 |
-
|
| 446 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 447 |
}
|
| 448 |
-
|
| 449 |
-
|
| 450 |
-
|
| 451 |
-
//
|
| 452 |
-
|
| 453 |
-
|
| 454 |
-
|
| 455 |
-
|
| 456 |
-
textInput.addEventListener('input', debouncedTokenize);
|
| 457 |
-
|
| 458 |
-
// Auto-load default model on page load
|
| 459 |
-
window.addEventListener('load', () => {
|
| 460 |
-
// Wait a bit for Transformers.js to fully initialize
|
| 461 |
-
setTimeout(loadModel, 500);
|
| 462 |
});
|
| 463 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 464 |
</body>
|
| 465 |
-
</html>
|
|
|
|
|
|
| 1 |
<!DOCTYPE html>
|
| 2 |
+
<html lang="en">
|
| 3 |
<head>
|
| 4 |
+
<meta charset="UTF-8">
|
| 5 |
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
| 6 |
+
<title>Universal Tokenizer Visualizer</title>
|
| 7 |
+
<script src="https://cdn.tailwindcss.com"></script>
|
| 8 |
+
<link href="https://cdn.jsdelivr.net/npm/@fontsource/inter@5.0.16/index.min.css" rel="stylesheet">
|
| 9 |
+
<style>
|
| 10 |
+
body { font-family: 'Inter', sans-serif; background: linear-gradient(135deg, #0f172a 0%, #1e293b 100%); color: #e2e8f0; }
|
| 11 |
+
.glass { background: rgba(30, 41, 59, 0.6); backdrop-filter: blur(12px); border: 1px solid rgba(255,255,255,0.08); }
|
| 12 |
+
.token-chip { transition: all 0.15s cubic-bezier(0.4, 0, 0.2, 1); cursor: default; position: relative; user-select: none; }
|
| 13 |
+
.token-chip:hover { transform: translateY(-3px) scale(1.05); z-index: 20; box-shadow: 0 8px 20px -4px rgba(0,0,0,0.5); }
|
| 14 |
+
.token-chip::after {
|
| 15 |
+
content: attr(data-tooltip);
|
| 16 |
+
position: absolute; bottom: 110%; left: 50%; transform: translateX(-50%) translateY(8px);
|
| 17 |
+
background: #0f172a; color: #cbd5e1; padding: 8px 12px; border-radius: 8px;
|
| 18 |
+
font-size: 0.75rem; line-height: 1.4; white-space: pre-wrap; max-width: 260px;
|
| 19 |
+
opacity: 0; pointer-events: none; transition: all 0.2s; border: 1px solid #334155;
|
| 20 |
+
box-shadow: 0 4px 12px rgba(0,0,0,0.4); z-index: 30; text-align: left;
|
| 21 |
+
}
|
| 22 |
+
.token-chip:hover::after { opacity: 1; transform: translateX(-50%) translateY(0); }
|
| 23 |
+
.spinner { width: 20px; height: 20px; border: 2px solid rgba(255,255,255,0.2); border-radius: 50%; border-top-color: #60a5fa; animation: spin 0.8s linear infinite; }
|
| 24 |
+
@keyframes spin { to { transform: rotate(360deg); } }
|
| 25 |
+
::-webkit-scrollbar { width: 8px; }
|
| 26 |
+
::-webkit-scrollbar-track { background: #1e293b; }
|
| 27 |
+
::-webkit-scrollbar-thumb { background: #475569; border-radius: 4px; }
|
| 28 |
+
.fade-in { animation: fadeIn 0.3s ease-out forwards; opacity: 0; }
|
| 29 |
+
@keyframes fadeIn { to { opacity: 1; transform: translateY(0); } }
|
| 30 |
+
.glow-btn { box-shadow: 0 0 20px rgba(59, 130, 246, 0.3); }
|
| 31 |
+
.glow-btn:hover { box-shadow: 0 0 30px rgba(59, 130, 246, 0.5); }
|
| 32 |
+
</style>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
</head>
|
| 34 |
+
<body class="min-h-screen flex flex-col items-center justify-start p-4 md:p-8">
|
| 35 |
+
|
| 36 |
+
<header class="w-full max-w-5xl mb-8 text-center md:text-left">
|
| 37 |
+
<h1 class="text-4xl font-bold bg-gradient-to-r from-cyan-400 via-blue-500 to-purple-500 bg-clip-text text-transparent tracking-tight">
|
| 38 |
+
🧩 Universal Tokenizer Visualizer
|
| 39 |
+
</h1>
|
| 40 |
+
<p class="text-slate-400 mt-2 text-sm md:text-base">
|
| 41 |
+
Load any HuggingFace tokenizer directly in your browser. 100% client-side, zero GPU cost.
|
| 42 |
+
</p>
|
| 43 |
+
</header>
|
| 44 |
+
|
| 45 |
+
<main class="w-full max-w-5xl grid grid-cols-1 lg:grid-cols-12 gap-6">
|
| 46 |
+
<!-- Control Panel -->
|
| 47 |
+
<section class="lg:col-span-4 glass rounded-2xl p-5 flex flex-col gap-4 shadow-2xl">
|
| 48 |
+
<div>
|
| 49 |
+
<label class="block text-sm font-medium text-slate-300 mb-1.5">HuggingFace Model ID</label>
|
| 50 |
+
<input type="text" id="modelId" value="gpt2"
|
| 51 |
+
class="w-full bg-slate-900/80 border border-slate-600 rounded-lg px-3 py-2.5 text-sm focus:outline-none focus:ring-2 focus:ring-blue-500/50 transition placeholder-slate-500"
|
| 52 |
+
placeholder="e.g., gpt2, meta-llama/Llama-3-8B">
|
| 53 |
+
<p class="text-xs text-slate-500 mt-1">Must contain <code class="bg-slate-800 px-1 rounded text-slate-300">tokenizer.json</code></p>
|
| 54 |
+
</div>
|
| 55 |
+
|
| 56 |
+
<div class="flex-grow">
|
| 57 |
+
<label class="block text-sm font-medium text-slate-300 mb-1.5">Input Text</label>
|
| 58 |
+
<textarea id="inputText" rows="7"
|
| 59 |
+
class="w-full bg-slate-900/80 border border-slate-600 rounded-lg px-3 py-2.5 text-sm focus:outline-none focus:ring-2 focus:ring-blue-500/50 resize-y transition placeholder-slate-500"
|
| 60 |
+
placeholder="Type or paste text to visualize..."></textarea>
|
| 61 |
+
</div>
|
| 62 |
+
|
| 63 |
+
<div class="flex flex-col gap-2 bg-slate-900/40 p-3 rounded-lg border border-slate-700/50">
|
| 64 |
+
<label class="flex items-center gap-2 cursor-pointer select-none">
|
| 65 |
+
<input type="checkbox" id="addSpecial" class="w-4 h-4 rounded bg-slate-800 border-slate-600 text-blue-500 focus:ring-blue-500" checked>
|
| 66 |
+
<span class="text-sm text-slate-300">Include special tokens (BOS/EOS)</span>
|
| 67 |
+
</label>
|
| 68 |
+
<label class="flex items-center gap-2 cursor-pointer select-none">
|
| 69 |
+
<input type="checkbox" id="showBytes" class="w-4 h-4 rounded bg-slate-800 border-slate-600 text-blue-500 focus:ring-blue-500">
|
| 70 |
+
<span class="text-sm text-slate-300">Show byte values for specials</span>
|
| 71 |
+
</label>
|
| 72 |
+
</div>
|
| 73 |
+
|
| 74 |
+
<button id="tokenizeBtn" class="w-full bg-blue-600 hover:bg-blue-500 active:scale-[0.98] text-white font-semibold py-2.5 px-4 rounded-lg transition flex items-center justify-center gap-2 disabled:opacity-50 disabled:cursor-not-allowed glow-btn">
|
| 75 |
+
<span>✨ Tokenize</span>
|
| 76 |
+
</button>
|
| 77 |
+
|
| 78 |
+
<div id="status" class="text-sm text-slate-400 h-6 flex items-center gap-2 truncate px-1"></div>
|
| 79 |
+
</section>
|
| 80 |
+
|
| 81 |
+
<!-- Visualization Panel -->
|
| 82 |
+
<section class="lg:col-span-8 glass rounded-2xl p-5 flex flex-col shadow-2xl min-h-[500px]">
|
| 83 |
+
<div class="flex flex-wrap justify-between items-center mb-4 pb-3 border-b border-slate-700/50 gap-3">
|
| 84 |
+
<h2 class="text-lg font-semibold text-slate-100">Token Breakdown</h2>
|
| 85 |
+
<div class="flex flex-wrap gap-3 text-sm">
|
| 86 |
+
<div class="flex items-center gap-1.5 bg-slate-800 px-3 py-1.5 rounded-md border border-slate-700">
|
| 87 |
+
<span class="w-2 h-2 rounded-full bg-blue-400"></span>
|
| 88 |
+
<span class="text-slate-300">Tokens: <b id="statTokens" class="text-white">0</b></span>
|
| 89 |
+
</div>
|
| 90 |
+
<div class="flex items-center gap-1.5 bg-slate-800 px-3 py-1.5 rounded-md border border-slate-700">
|
| 91 |
+
<span class="w-2 h-2 rounded-full bg-purple-400"></span>
|
| 92 |
+
<span class="text-slate-300">Chars: <b id="statChars" class="text-white">0</b></span>
|
| 93 |
+
</div>
|
| 94 |
+
<div class="flex items-center gap-1.5 bg-slate-800 px-3 py-1.5 rounded-md border border-slate-700">
|
| 95 |
+
<span class="w-2 h-2 rounded-full bg-pink-400"></span>
|
| 96 |
+
<span class="text-slate-300">Ratio: <b id="statRatio" class="text-white">0.00</b></span>
|
| 97 |
+
</div>
|
| 98 |
+
</div>
|
| 99 |
+
</div>
|
| 100 |
+
|
| 101 |
+
<div id="outputGrid" class="flex flex-wrap gap-2 overflow-y-auto p-3 content-start flex-grow min-h-[350px] bg-slate-900/30 rounded-xl border border-slate-800/40">
|
| 102 |
+
<div class="w-full h-full flex flex-col items-center justify-center text-slate-500 select-none py-12">
|
| 103 |
+
<svg class="w-14 h-14 mb-4 opacity-30" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="1.5" d="M13 10V3L4 14h7v7l9-11h-7z"></path></svg>
|
| 104 |
+
<p class="mb-2">Enter text and click <span class="text-blue-400 font-medium">Tokenize</span> to visualize.</p>
|
| 105 |
+
<p class="text-xs opacity-60">Try <code>gpt2</code>, <code>bert-base-uncased</code>, or <code>Qwen/Qwen2.5-7B</code></p>
|
| 106 |
+
</div>
|
| 107 |
+
</div>
|
| 108 |
+
</section>
|
| 109 |
+
</main>
|
| 110 |
+
|
| 111 |
+
<footer class="mt-10 text-center text-slate-600 text-xs flex flex-col md:flex-row items-center gap-2 pb-6">
|
| 112 |
+
<span>Powered by <code class="bg-slate-800 px-1.5 py-0.5 rounded text-blue-400">@huggingface/transformers@3</code></span>
|
| 113 |
+
<span>•</span>
|
| 114 |
+
<span>Runs entirely in-browser via WebAssembly</span>
|
| 115 |
+
<span>•</span>
|
| 116 |
+
<a href="https://github.com/huggingface/transformers.js" target="_blank" class="text-blue-400 hover:underline">Docs</a>
|
| 117 |
+
</footer>
|
| 118 |
+
|
| 119 |
+
<script type="module">
|
| 120 |
+
// Import transformers.js from CDN (ESM compatible)
|
| 121 |
+
import { AutoTokenizer } from 'https://esm.sh/@huggingface/transformers@3.0.2';
|
| 122 |
+
|
| 123 |
+
const COLORS = [
|
| 124 |
+
'#3b82f6', '#8b5cf6', '#ec4899', '#f59e0b', '#10b981',
|
| 125 |
+
'#06b6d4', '#6366f1', '#f97316', '#84cc16', '#14b8a6', '#f43f5e', '#a855f7'
|
| 126 |
+
];
|
| 127 |
+
|
| 128 |
+
const els = {
|
| 129 |
+
modelId: document.getElementById('modelId'),
|
| 130 |
+
inputText: document.getElementById('inputText'),
|
| 131 |
+
btn: document.getElementById('tokenizeBtn'),
|
| 132 |
+
output: document.getElementById('outputGrid'),
|
| 133 |
+
status: document.getElementById('status'),
|
| 134 |
+
statTokens: document.getElementById('statTokens'),
|
| 135 |
+
statChars: document.getElementById('statChars'),
|
| 136 |
+
statRatio: document.getElementById('statRatio'),
|
| 137 |
+
addSpecial: document.getElementById('addSpecial'),
|
| 138 |
+
showBytes: document.getElementById('showBytes')
|
| 139 |
+
};
|
| 140 |
+
|
| 141 |
+
let tokenizerCache = null;
|
| 142 |
+
|
| 143 |
+
function getColor(idx) { return COLORS[idx % COLORS.length]; }
|
| 144 |
+
|
| 145 |
+
function setStatus(msg, type = 'info') {
|
| 146 |
+
els.status.textContent = msg;
|
| 147 |
+
els.status.className = `text-sm h-6 flex items-center gap-2 truncate px-1 ${
|
| 148 |
+
type === 'error' ? 'text-red-400' :
|
| 149 |
+
type === 'success' ? 'text-green-400' :
|
| 150 |
+
'text-slate-400'
|
| 151 |
+
}`;
|
| 152 |
+
}
|
| 153 |
+
|
| 154 |
+
async function loadTokenizer(modelId) {
|
| 155 |
+
if (tokenizerCache && tokenizerCache._id === modelId) return tokenizerCache.tokenizer;
|
| 156 |
+
|
| 157 |
+
setStatus('📥 Fetching tokenizer configuration...', 'info');
|
| 158 |
+
try {
|
| 159 |
+
// Progress callback for better UX on large tokenizers
|
| 160 |
+
const tokenizer = await AutoTokenizer.from_pretrained(modelId, {
|
| 161 |
+
progress_callback: (p) => {
|
| 162 |
+
if (p.status === 'loading' || p.status === 'downloading') {
|
| 163 |
+
const pct = Math.round((p.loaded / p.total) * 100);
|
| 164 |
+
setStatus(`📥 ${pct}% ${p.file || 'loading'}`, 'info');
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 165 |
}
|
| 166 |
+
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 167 |
});
|
| 168 |
+
|
| 169 |
+
tokenizerCache = { _id: modelId, tokenizer };
|
| 170 |
+
return tokenizer;
|
| 171 |
+
} catch (err) {
|
| 172 |
+
console.error(err);
|
| 173 |
+
throw new Error(`Failed to load tokenizer. Check Model ID, network, or repo visibility. (${err.message})`);
|
| 174 |
+
}
|
| 175 |
+
}
|
| 176 |
+
|
| 177 |
+
function getBackend(tokenizer) {
|
| 178 |
+
// transformers.js v3 exposes backend_tokenizer
|
| 179 |
+
// Fallback to common property names if structure changes
|
| 180 |
+
return tokenizer.backend_tokenizer ?? tokenizer.tokenizer ?? tokenizer;
|
| 181 |
+
}
|
| 182 |
+
|
| 183 |
+
els.btn.addEventListener('click', async () => {
|
| 184 |
+
const modelId = els.modelId.value.trim();
|
| 185 |
+
const text = els.inputText.value;
|
| 186 |
+
const addSpecial = els.addSpecial.checked;
|
| 187 |
+
const showBytes = els.showBytes.checked;
|
| 188 |
+
|
| 189 |
+
if (!modelId) return setStatus('⚠️ Please enter a Model ID', 'error');
|
| 190 |
+
if (!text) return setStatus('⚠️ Please enter text to tokenize', 'error');
|
| 191 |
+
|
| 192 |
+
els.btn.disabled = true;
|
| 193 |
+
els.btn.innerHTML = '<div class="spinner"></div> <span class="ml-2">Loading...</span>';
|
| 194 |
+
els.output.innerHTML = '';
|
| 195 |
+
setStatus('🔄 Initializing tokenizer...');
|
| 196 |
+
|
| 197 |
+
try {
|
| 198 |
+
const tokenizer = await loadTokenizer(modelId);
|
| 199 |
+
setStatus('⚡ Running tokenization...', 'info');
|
| 200 |
+
|
| 201 |
+
const backend = getBackend(tokenizer);
|
| 202 |
+
if (!backend || typeof backend.encode !== 'function') {
|
| 203 |
+
throw new Error('Backend tokenizer encoding method not found. This model format might not be supported in-browser.');
|
| 204 |
}
|
| 205 |
+
|
| 206 |
+
// encode returns { tokens, ids, offsets, ... }
|
| 207 |
+
const result = backend.encode(text, { add_special_tokens: addSpecial });
|
| 208 |
+
renderTokens(result.tokens, result.ids, result.offsets, text, showBytes);
|
| 209 |
+
setStatus(`✨ Done • ${result.ids.length} tokens loaded`, 'success');
|
| 210 |
+
} catch (err) {
|
| 211 |
+
console.error('Tokenization error:', err);
|
| 212 |
+
setStatus(`❌ ${err.message}`, 'error');
|
| 213 |
+
els.output.innerHTML = `
|
| 214 |
+
<div class="w-full text-center py-12">
|
| 215 |
+
<div class="text-red-400 font-medium mb-2">Tokenization Failed</div>
|
| 216 |
+
<div class="text-slate-500 text-sm max-w-md mx-auto space-y-2">
|
| 217 |
+
<p>${err.message}</p>
|
| 218 |
+
<div class="bg-slate-800/50 p-3 rounded border border-slate-700 text-left mt-4">
|
| 219 |
+
<p class="text-slate-400 mb-1">Troubleshooting:</p>
|
| 220 |
+
<ul class="list-disc list-inside space-y-1 text-xs text-slate-500">
|
| 221 |
+
<li>Try a base model first: <code>gpt2</code>, <code>bert-base-uncased</code></li>
|
| 222 |
+
<li>Ensure <code>tokenizer.json</code> exists in the repo</li>
|
| 223 |
+
<li>Custom tokenizers (e.g., DeepSeek-V3) may need specific config</li>
|
| 224 |
+
<li>Check browser console for network/CORS errors</li>
|
| 225 |
+
</ul>
|
| 226 |
+
</div>
|
| 227 |
+
</div>
|
| 228 |
+
</div>`;
|
| 229 |
+
} finally {
|
| 230 |
+
els.btn.disabled = false;
|
| 231 |
+
els.btn.innerHTML = '<span>✨ Tokenize</span>';
|
| 232 |
+
}
|
| 233 |
+
});
|
| 234 |
+
|
| 235 |
+
function renderTokens(tokens, ids, offsets, originalText, showBytes) {
|
| 236 |
+
els.output.innerHTML = '';
|
| 237 |
+
const count = ids.length;
|
| 238 |
+
els.statTokens.textContent = count;
|
| 239 |
+
els.statChars.textContent = originalText.length;
|
| 240 |
+
els.statRatio.textContent = originalText.length > 0 ? (count / originalText.length).toFixed(2) : '0.00';
|
| 241 |
+
|
| 242 |
+
tokens.forEach((tok, i) => {
|
| 243 |
+
const [start, end] = offsets[i] || [0, 0];
|
| 244 |
+
const chunk = originalText.slice(start, end);
|
| 245 |
+
const color = getColor(i);
|
| 246 |
+
const id = ids[i];
|
| 247 |
+
|
| 248 |
+
const chip = document.createElement('div');
|
| 249 |
+
chip.className = 'token-chip rounded-md px-2 py-1.5 text-sm font-mono border select-none fade-in';
|
| 250 |
+
chip.style.animationDelay = `${Math.min(i * 0.02, 2)}s`; // Cap animation delay
|
| 251 |
+
chip.style.animationDelay = `${i * 0.015}s`;
|
| 252 |
+
chip.style.backgroundColor = `${color}15`;
|
| 253 |
+
chip.style.borderColor = `${color}80`;
|
| 254 |
+
chip.style.color = color;
|
| 255 |
+
|
| 256 |
+
// Tooltip details
|
| 257 |
+
const cleanChunk = chunk.replace(/"/g, '\\"').replace(/\n/g, '\\n').replace(/\t/g, '\\t');
|
| 258 |
+
const byteInfo = showBytes ? `\nBytes: ${[...chunk].map(c => '0x'+c.charCodeAt(0).toString(16).padStart(2,'0')).join(' ')}` : '';
|
| 259 |
+
chip.dataset.tooltip = `ID: ${id}\nToken: "${tok}"\nOffset: [${start}, ${end})\nText: "${cleanChunk}"${byteInfo}`;
|
| 260 |
+
|
| 261 |
+
// Visual handling of special/invisible tokens
|
| 262 |
+
let displayText = tok;
|
| 263 |
+
if (tok.startsWith('<') && tok.endsWith('>')) {
|
| 264 |
+
chip.classList.add('border-dashed', 'opacity-80');
|
| 265 |
+
} else if (chunk === '') {
|
| 266 |
+
displayText = '␣';
|
| 267 |
+
} else if (chunk === '\n') {
|
| 268 |
+
displayText = '↵';
|
| 269 |
+
} else if (chunk === '\t') {
|
| 270 |
+
displayText = '→';
|
| 271 |
}
|
| 272 |
+
|
| 273 |
+
chip.innerHTML = `<span class="font-medium">${displayText}</span><span class="opacity-60 text-[10px] ml-1.5 align-top font-sans">${id}</span>`;
|
| 274 |
+
|
| 275 |
+
// Interactive highlight
|
| 276 |
+
chip.addEventListener('click', () => {
|
| 277 |
+
document.querySelectorAll('.token-chip').forEach(c => c.style.boxShadow = 'none');
|
| 278 |
+
chip.style.boxShadow = `0 0 0 2px ${color}, 0 4px 12px ${color}40`;
|
| 279 |
+
// Optional: auto-scroll to text or highlight in textarea
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 280 |
});
|
| 281 |
+
|
| 282 |
+
els.output.appendChild(chip);
|
| 283 |
+
});
|
| 284 |
+
}
|
| 285 |
+
|
| 286 |
+
// Keyboard shortcut: Ctrl/Cmd + Enter
|
| 287 |
+
document.addEventListener('keydown', (e) => {
|
| 288 |
+
if ((e.ctrlKey || e.metaKey) && e.key === 'Enter') {
|
| 289 |
+
els.btn.click();
|
| 290 |
+
}
|
| 291 |
+
});
|
| 292 |
+
|
| 293 |
+
// Init demo
|
| 294 |
+
window.addEventListener('load', () => {
|
| 295 |
+
if (!els.inputText.value) {
|
| 296 |
+
els.inputText.value = "Tokenization is the bridge between human language and AI. It breaks text into numerical pieces! 🚀";
|
| 297 |
+
}
|
| 298 |
+
});
|
| 299 |
+
</script>
|
| 300 |
</body>
|
| 301 |
+
</html>
|
| 302 |
+
|