quickgrid commited on
Commit
6a8f8b2
·
verified ·
1 Parent(s): 2ccc681
Files changed (1) hide show
  1. index.html +628 -977
index.html CHANGED
@@ -1,5 +1,5 @@
1
  <!DOCTYPE html>
2
- <html lang="en">
3
  <head>
4
  <meta charset="UTF-8" />
5
  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
@@ -7,7 +7,6 @@
7
  <meta name="description" content="Visualize how large language models tokenize text. Powered by Transformers.js, runs entirely in your browser." />
8
  <link rel="preconnect" href="https://fonts.googleapis.com" />
9
  <link href="https://fonts.googleapis.com/css2?family=Bricolage+Grotesque:opsz,wght@12..96,300;12..96,400;12..96,500;12..96,600;12..96,700;12..96,800&family=JetBrains+Mono:wght@300;400;500;700&family=DM+Sans:wght@300;400;500&display=swap" rel="stylesheet" />
10
-
11
  <style>
12
  /* ─── Design Tokens ─────────────────────────────────── */
13
  :root {
@@ -26,792 +25,464 @@
26
  --green: #34d89a;
27
  --amber: #f5a623;
28
  --red: #f55577;
29
-
30
- /* Token palette — 14 vivid colors for dark bg */
31
- --t0: #ff8080; --t0b: rgba(255,128,128,.18);
32
- --t1: #ffb84d; --t1b: rgba(255,184, 77,.18);
33
- --t2: #ffe066; --t2b: rgba(255,224,102,.18);
34
- --t3: #7aed91; --t3b: rgba(122,237,145,.18);
35
- --t4: #4ddfc0; --t4b: rgba( 77,223,192,.18);
36
- --t5: #56c8f5; --t5b: rgba( 86,200,245,.18);
37
- --t6: #748ef8; --t6b: rgba(116,142,248,.18);
38
- --t7: #c484f8; --t7b: rgba(196,132,248,.18);
39
- --t8: #f57cd4; --t8b: rgba(245,124,212,.18);
40
- --t9: #fa8072; --t9b: rgba(250,128,114,.18);
41
- --t10: #8be08b; --t10b: rgba(139,224,139,.18);
42
- --t11: #f0c040; --t11b: rgba(240,192, 64,.18);
43
- --t12: #60d4e0; --t12b: rgba( 96,212,224,.18);
44
- --t13: #e89060; --t13b: rgba(232,144, 96,.18);
45
  }
46
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
47
  /* ─── Reset ─────────────────────────────────────────── */
48
  *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
49
-
50
- html { scroll-behavior: smooth; }
51
-
52
  body {
53
  background: var(--bg);
54
  color: var(--text);
55
  font-family: 'DM Sans', sans-serif;
56
- min-height: 100vh;
57
- display: flex;
58
- flex-direction: column;
59
- overflow-x: hidden;
60
  }
61
-
62
  /* ─── Background FX ─────────────────────────────────── */
63
- #bg-canvas {
64
- position: fixed;
65
- inset: 0;
66
- pointer-events: none;
67
- z-index: 0;
68
- }
69
  .bg-gradient {
70
- position: fixed;
71
- inset: 0;
72
- pointer-events: none;
73
- z-index: 0;
74
  background:
75
  radial-gradient(ellipse 80% 50% at 20% 10%, rgba(77,158,245,.06) 0%, transparent 70%),
76
  radial-gradient(ellipse 60% 40% at 80% 90%, rgba(139,106,245,.05) 0%, transparent 60%),
77
  radial-gradient(ellipse 40% 30% at 60% 50%, rgba(52,216,154,.03) 0%, transparent 60%);
78
  }
 
 
 
 
 
 
79
  .dot-grid {
80
- position: fixed;
81
- inset: 0;
82
- pointer-events: none;
83
- z-index: 0;
84
  background-image: radial-gradient(circle, rgba(77,158,245,.12) 1px, transparent 1px);
85
  background-size: 36px 36px;
86
  mask-image: radial-gradient(ellipse 100% 100% at 50% 50%, black 30%, transparent 80%);
87
  }
88
-
 
 
89
  /* ─── Layout ─────────────────────────────────────────── */
90
  #app {
91
- position: relative;
92
- z-index: 1;
93
- display: flex;
94
- flex-direction: column;
95
- min-height: 100vh;
96
  }
97
-
98
  /* ─── Header ─────────────────────────────────────────── */
99
  header {
100
- display: flex;
101
- align-items: center;
102
- justify-content: space-between;
103
- padding: 0 32px;
104
- height: 64px;
105
  border-bottom: 1px solid var(--border);
106
  background: rgba(6,11,20,.85);
107
  backdrop-filter: blur(20px);
108
- position: sticky;
109
- top: 0;
110
- z-index: 100;
111
- gap: 16px;
112
  }
113
-
114
  .logo {
115
- display: flex;
116
- align-items: center;
117
- gap: 10px;
118
- text-decoration: none;
119
- color: var(--text);
120
- flex-shrink: 0;
121
  }
122
  .logo-hex {
123
- width: 34px;
124
- height: 34px;
125
  background: linear-gradient(135deg, var(--accent), var(--accent2));
126
  clip-path: polygon(50% 0%, 93% 25%, 93% 75%, 50% 100%, 7% 75%, 7% 25%);
127
- display: flex;
128
- align-items: center;
129
- justify-content: center;
130
- font-size: 14px;
131
- font-family: 'JetBrains Mono', monospace;
132
- font-weight: 700;
133
- color: white;
134
  }
135
  .logo-name {
136
  font-family: 'Bricolage Grotesque', sans-serif;
137
- font-size: 20px;
138
- font-weight: 700;
139
- letter-spacing: -0.5px;
140
  background: linear-gradient(135deg, #dce8f8 40%, var(--accent));
141
- -webkit-background-clip: text;
142
- -webkit-text-fill-color: transparent;
143
- background-clip: text;
144
  }
145
- .logo-tag {
146
- font-size: 10px;
147
- font-family: 'JetBrains Mono', monospace;
148
- color: var(--text3);
149
- background: var(--bg3);
150
- border: 1px solid var(--border);
151
- padding: 2px 6px;
152
- border-radius: 4px;
153
- letter-spacing: .5px;
154
- }
155
-
156
- /* Model tabs in header */
157
- .model-tabs {
158
- display: flex;
159
- gap: 6px;
160
- align-items: center;
161
- flex-shrink: 0;
162
- }
163
- .model-tab {
164
- display: flex;
165
- flex-direction: column;
166
- padding: 6px 10px;
167
- border: 1px solid var(--border);
168
- border-radius: 8px;
169
- background: var(--bg2);
170
- cursor: pointer;
171
- transition: all 0.2s ease;
172
- position: relative;
173
- overflow: hidden;
174
- min-width: 90px;
175
  }
176
- .model-tab::before {
177
- content: '';
178
- position: absolute;
179
- inset: 0;
180
- background: linear-gradient(135deg, var(--accent), var(--accent2));
181
- opacity: 0;
182
- transition: opacity 0.2s;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
183
  }
184
- .model-tab:hover {
185
- border-color: var(--border2);
186
- transform: translateY(-1px);
 
 
 
187
  }
188
- .model-tab.active {
 
189
  border-color: var(--accent);
190
- box-shadow: 0 0 0 1px var(--accent), 0 0 20px rgba(77,158,245,.15);
191
- }
192
- .model-tab.active::before { opacity: .08; }
193
- .model-tab-name {
194
- font-family: 'Bricolage Grotesque', sans-serif;
195
- font-size: 13px;
196
- font-weight: 600;
197
- color: var(--text);
198
- position: relative;
199
- }
200
- .model-tab-org {
201
- font-size: 10px;
202
- color: var(--text2);
203
- font-family: 'JetBrains Mono', monospace;
204
- position: relative;
205
- margin-top: 1px;
206
- }
207
- .model-tab-vocab {
208
- font-size: 10px;
209
- color: var(--text3);
210
- font-family: 'JetBrains Mono', monospace;
211
- position: relative;
212
- }
213
- .model-org-dot {
214
- width: 6px;
215
- height: 6px;
216
- border-radius: 50%;
217
- display: inline-block;
218
- margin-right: 4px;
219
- position: relative;
220
- top: -1px;
221
  }
222
-
223
- /* Custom model row in header */
224
- .custom-model-row {
225
- display: flex;
226
- align-items: center;
227
- gap: 8px;
228
- flex-shrink: 0;
229
- }
230
- .custom-model-row label {
231
- font-size: 11px;
232
- color: var(--text2);
233
- font-family: 'JetBrains Mono', monospace;
234
- white-space: nowrap;
235
- }
236
- .custom-input {
237
- width: 180px;
238
- max-width: 220px;
239
- background: var(--bg2);
240
- border: 1px solid var(--border);
241
- border-radius: 8px;
242
- color: var(--text);
243
- font-family: 'JetBrains Mono', monospace;
244
- font-size: 13px;
245
- padding: 6px 10px;
246
- outline: none;
247
- transition: border-color 0.2s;
248
- }
249
- .custom-input:focus { border-color: var(--accent); }
250
- .custom-input::placeholder { color: var(--text3); }
251
- .btn {
252
- padding: 6px 14px;
253
- border-radius: 8px;
254
- border: 1px solid var(--border2);
255
- background: linear-gradient(135deg, rgba(77,158,245,.15), rgba(139,106,245,.15));
256
- color: var(--accent);
257
- font-family: 'DM Sans', sans-serif;
258
- font-size: 13px;
259
- font-weight: 500;
260
- cursor: pointer;
261
- transition: all 0.2s;
262
- white-space: nowrap;
263
- }
264
- .btn:hover {
265
- background: linear-gradient(135deg, rgba(77,158,245,.25), rgba(139,106,245,.25));
266
- border-color: var(--accent);
267
- }
268
- .btn:active { transform: scale(.97); }
269
-
270
  /* ─── Main Split ─────────────────────────────────────── */
271
  main {
272
- flex: 1;
273
- display: grid;
274
- grid-template-columns: 1fr 1fr;
275
- gap: 0;
276
- min-height: 0;
277
  }
278
-
279
  /* ─── Left Panel (Input) ─────────────────────────────── */
280
  .input-panel {
281
  border-right: 1px solid var(--border);
282
- display: flex;
283
- flex-direction: column;
284
- padding: 0;
285
  }
286
  .panel-header {
287
- padding: 16px 24px 12px;
288
- border-bottom: 1px solid var(--border);
289
- display: flex;
290
- align-items: center;
291
- justify-content: space-between;
292
  }
293
  .panel-title {
294
- font-family: 'Bricolage Grotesque', sans-serif;
295
- font-size: 14px;
296
- font-weight: 600;
297
- color: var(--text2);
298
- letter-spacing: .3px;
299
- display: flex;
300
- align-items: center;
301
- gap: 8px;
302
  }
303
  .panel-title-icon {
304
- width: 20px;
305
- height: 20px;
306
- background: var(--bg4);
307
- border: 1px solid var(--border);
308
- border-radius: 5px;
309
- display: flex;
310
- align-items: center;
311
- justify-content: center;
312
- font-size: 11px;
313
- }
314
-
315
- .sample-btns {
316
- display: flex;
317
- gap: 6px;
318
  }
 
319
  .sample-btn {
320
- font-size: 11px;
321
- padding: 4px 10px;
322
- border-radius: 6px;
323
- border: 1px solid var(--border);
324
- background: var(--bg2);
325
- color: var(--text2);
326
- cursor: pointer;
327
- font-family: 'DM Sans', sans-serif;
328
- transition: all .15s;
329
  }
330
- .sample-btn:hover {
331
- border-color: var(--border2);
332
- color: var(--text);
333
- }
334
-
335
  #input-area {
336
- flex: 1;
337
- width: 100%;
338
- background: transparent;
339
- border: none;
340
- outline: none;
341
- resize: none;
342
- color: var(--text);
343
- font-family: 'DM Sans', sans-serif;
344
- font-size: 15px;
345
- line-height: 1.7;
346
- padding: 20px 24px;
347
- min-height: 220px;
348
  }
349
  #input-area::placeholder { color: var(--text3); }
350
-
351
  .char-counter {
352
- padding: 8px 24px;
353
- border-top: 1px solid var(--border);
354
- font-size: 11px;
355
- font-family: 'JetBrains Mono', monospace;
356
- color: var(--text3);
357
- text-align: right;
358
  }
359
-
360
- /* ─── Right Panel (Output) ───────────────────────────── */
361
  .output-panel {
362
- display: flex;
363
- flex-direction: column;
364
- overflow: hidden;
365
  }
366
-
 
 
 
 
 
 
 
 
 
 
367
  /* Stats row */
368
  .stats-row {
369
- display: grid;
370
- grid-template-columns: repeat(4, 1fr);
371
- border-bottom: 1px solid var(--border);
372
  }
373
  .stat-card {
374
- padding: 16px 20px;
375
- border-right: 1px solid var(--border);
376
- position: relative;
377
- overflow: hidden;
378
  }
379
- .stat-card:last-child { border-right: none; }
 
380
  .stat-card::after {
381
- content: '';
382
- position: absolute;
383
- bottom: 0;
384
- left: 0;
385
- right: 0;
386
- height: 2px;
387
  background: linear-gradient(90deg, transparent, var(--accent), transparent);
388
- opacity: 0;
389
- transition: opacity .3s;
390
  }
391
  .stat-card.highlight::after { opacity: 1; }
392
  .stat-label {
393
- font-size: 10px;
394
- font-family: 'JetBrains Mono', monospace;
395
- color: var(--text3);
396
- text-transform: uppercase;
397
- letter-spacing: 1px;
398
- margin-bottom: 6px;
399
  }
400
  .stat-value {
401
- font-family: 'Bricolage Grotesque', sans-serif;
402
- font-size: 26px;
403
- font-weight: 700;
404
- color: var(--text);
405
- line-height: 1;
406
- transition: all .3s;
407
  }
408
  .stat-card:nth-child(1) .stat-value { color: var(--accent); }
409
  .stat-card:nth-child(2) .stat-value { color: var(--green); }
410
  .stat-card:nth-child(3) .stat-value { color: var(--amber); }
411
  .stat-card:nth-child(4) .stat-value { color: var(--accent2); }
412
  .stat-sub {
413
- font-size: 10px;
414
- color: var(--text3);
415
- font-family: 'JetBrains Mono', monospace;
416
- margin-top: 3px;
417
  }
418
-
419
  /* View toggle */
420
  .view-toggle {
421
- display: flex;
422
- padding: 12px 20px;
423
- border-bottom: 1px solid var(--border);
424
- gap: 4px;
425
- align-items: center;
426
- justify-content: space-between;
427
  }
428
  .toggle-group {
429
- display: flex;
430
- gap: 4px;
431
- background: var(--bg2);
432
- border: 1px solid var(--border);
433
- border-radius: 8px;
434
- padding: 3px;
435
  }
436
  .toggle-btn {
437
- padding: 5px 14px;
438
- border-radius: 6px;
439
- border: none;
440
- background: transparent;
441
- color: var(--text2);
442
- font-family: 'DM Sans', sans-serif;
443
- font-size: 12px;
444
- font-weight: 500;
445
- cursor: pointer;
446
- transition: all .15s;
447
  }
448
- .toggle-btn.active {
449
- background: var(--bg4);
450
- color: var(--text);
451
- box-shadow: 0 1px 4px rgba(0,0,0,.3);
452
- }
453
- .special-toggle {
454
- display: flex;
455
- align-items: center;
456
- gap: 8px;
457
- font-size: 12px;
458
- color: var(--text2);
459
- }
460
- .toggle-switch {
461
- width: 32px;
462
- height: 18px;
463
- background: var(--bg4);
464
- border: 1px solid var(--border);
465
- border-radius: 9px;
466
- cursor: pointer;
467
- position: relative;
468
- transition: background .2s;
469
- }
470
- .toggle-switch::after {
471
- content: '';
472
- position: absolute;
473
- width: 12px;
474
- height: 12px;
475
- border-radius: 50%;
476
- background: var(--text3);
477
- top: 2px;
478
- left: 2px;
479
- transition: all .2s;
480
- }
481
- .toggle-switch.on { background: rgba(77,158,245,.3); border-color: var(--accent); }
482
- .toggle-switch.on::after { left: 16px; background: var(--accent); }
483
-
484
  /* Token Display */
485
  .token-display {
486
- flex: 1;
487
- overflow-y: auto;
488
- padding: 20px;
489
- scrollbar-width: thin;
490
- scrollbar-color: var(--border) transparent;
491
  }
492
-
493
  .placeholder-msg {
494
- display: flex;
495
- flex-direction: column;
496
- align-items: center;
497
- justify-content: center;
498
- height: 200px;
499
- gap: 16px;
500
- color: var(--text3);
501
- }
502
- .placeholder-icon {
503
- font-size: 40px;
504
- filter: grayscale(1) opacity(.3);
505
  }
 
506
  .placeholder-msg p {
507
- font-family: 'JetBrains Mono', monospace;
508
- font-size: 13px;
509
- text-align: center;
510
  }
511
-
512
  /* ─── Token Visualization Views ───────────────────────── */
513
-
514
- /* TEXT VIEW — inline colored token spans */
515
  .token-text-view {
516
- font-family: 'JetBrains Mono', monospace;
517
- font-size: 14px;
518
- line-height: 2.2;
519
- word-break: break-all;
520
  }
521
  .tok {
522
- display: inline;
523
- border-radius: 4px;
524
- padding: 1px 0;
525
- cursor: default;
526
- transition: filter .15s;
527
- position: relative;
528
  }
529
  .tok:hover { filter: brightness(1.3); }
530
  .tok-tooltip {
531
- display: none;
532
- position: absolute;
533
- bottom: 110%;
534
- left: 50%;
535
- transform: translateX(-50%);
536
- background: var(--bg4);
537
- border: 1px solid var(--border2);
538
- border-radius: 6px;
539
- padding: 5px 8px;
540
- font-size: 11px;
541
- white-space: nowrap;
542
- z-index: 50;
543
- pointer-events: none;
544
- box-shadow: 0 4px 20px rgba(0,0,0,.5);
545
  }
 
546
  .tok:hover .tok-tooltip { display: block; }
547
  .tok-tooltip-id { color: var(--accent); font-weight: 700; }
548
  .tok-tooltip-text { color: var(--text2); }
549
  .tok-space::before { content: '·'; opacity: .3; }
550
  .tok-newline::before { content: '↵'; opacity: .5; }
551
-
552
- /* ID VIEW grid of token cards */
553
- .token-id-view {
554
- display: flex;
555
- flex-wrap: wrap;
556
- gap: 6px;
557
- }
558
  .tok-id-card {
559
- display: flex;
560
- flex-direction: column;
561
- align-items: center;
562
- border-radius: 8px;
563
- overflow: hidden;
564
- border: 1px solid;
565
- cursor: default;
566
- transition: transform .15s, box-shadow .15s;
567
- min-width: 52px;
568
- }
569
- .tok-id-card:hover {
570
- transform: translateY(-2px);
571
- box-shadow: 0 4px 16px rgba(0,0,0,.4);
572
  }
 
573
  .tok-id-top {
574
- padding: 3px 6px;
575
- font-family: 'JetBrains Mono', monospace;
576
- font-size: 11px;
577
- font-weight: 500;
578
- width: 100%;
579
- text-align: center;
580
- border-bottom: 1px solid rgba(255,255,255,.08);
581
  }
 
582
  .tok-id-bottom {
583
- padding: 2px 6px 3px;
584
- font-family: 'JetBrains Mono', monospace;
585
- font-size: 9px;
586
- color: rgba(255,255,255,.4);
587
- width: 100%;
588
- text-align: center;
589
- }
590
-
591
- /* PROBABILITY VIEW placeholder */
592
- .token-split-view {
593
- display: flex;
594
- flex-direction: column;
595
- gap: 3px;
596
  }
 
 
 
597
  .tok-split-row {
598
- display: flex;
599
- align-items: stretch;
600
- border-radius: 6px;
601
- overflow: hidden;
602
- border: 1px solid;
603
- font-family: 'JetBrains Mono', monospace;
604
- font-size: 12px;
605
  }
606
  .tok-split-idx {
607
- width: 38px;
608
- text-align: center;
609
- padding: 5px 4px;
610
- font-size: 10px;
611
- color: rgba(255,255,255,.3);
612
- border-right: 1px solid rgba(255,255,255,.06);
613
- display: flex;
614
- align-items: center;
615
- justify-content: center;
616
- }
617
- .tok-split-text {
618
- flex: 1;
619
- padding: 5px 8px;
620
- font-size: 13px;
621
  }
 
 
622
  .tok-split-id {
623
- padding: 5px 8px;
624
- font-size: 11px;
625
- color: rgba(255,255,255,.45);
626
- border-left: 1px solid rgba(255,255,255,.06);
627
- display: flex;
628
- align-items: center;
629
  }
630
-
631
  /* ─── Loading Overlay ────────────────────────────────── */
632
  #loading-overlay {
633
- position: fixed;
634
- inset: 0;
635
- background: rgba(6,11,20,.92);
636
- backdrop-filter: blur(8px);
637
- z-index: 1000;
638
- display: flex;
639
- flex-direction: column;
640
- align-items: center;
641
- justify-content: center;
642
- gap: 24px;
643
- transition: opacity .4s;
644
  }
 
645
  #loading-overlay.hidden { opacity: 0; pointer-events: none; }
646
-
647
- .loading-spinner {
648
- width: 56px;
649
- height: 56px;
650
- position: relative;
651
- }
652
- .loading-spinner::before,
653
- .loading-spinner::after {
654
- content: '';
655
- position: absolute;
656
- border-radius: 50%;
657
- border: 2px solid transparent;
658
- }
659
- .loading-spinner::before {
660
- inset: 0;
661
- border-top-color: var(--accent);
662
- animation: spin 1s linear infinite;
663
- }
664
- .loading-spinner::after {
665
- inset: 8px;
666
- border-top-color: var(--accent2);
667
- animation: spin .7s linear infinite reverse;
668
  }
 
 
669
  @keyframes spin { to { transform: rotate(360deg); } }
670
-
671
- .loading-text {
672
- font-family: 'Bricolage Grotesque', sans-serif;
673
- font-size: 20px;
674
- font-weight: 600;
675
- color: var(--text);
676
- }
677
  .loading-sub {
678
- font-family: 'JetBrains Mono', monospace;
679
- font-size: 12px;
680
- color: var(--text2);
681
- max-width: 360px;
682
- text-align: center;
683
- }
684
- .loading-bar-wrap {
685
- width: 300px;
686
- height: 3px;
687
- background: var(--bg3);
688
- border-radius: 2px;
689
- overflow: hidden;
690
  }
 
691
  .loading-bar {
692
- height: 100%;
693
- width: 0%;
694
  background: linear-gradient(90deg, var(--accent), var(--accent2));
695
- border-radius: 2px;
696
- transition: width .3s;
697
- }
698
- .loading-file {
699
- font-size: 11px;
700
- font-family: 'JetBrains Mono', monospace;
701
- color: var(--text3);
702
  }
703
-
704
  /* ─── Error Toast ────────────────────────────────────── */
705
  #toast {
706
- position: fixed;
707
- bottom: 24px;
708
- left: 50%;
709
  transform: translateX(-50%) translateY(80px);
710
- background: rgba(245,85,119,.15);
711
- border: 1px solid rgba(245,85,119,.4);
712
- color: var(--red);
713
- padding: 10px 20px;
714
- border-radius: 10px;
715
- font-size: 13px;
716
- font-family: 'JetBrains Mono', monospace;
717
- z-index: 500;
718
- transition: transform .3s;
719
- max-width: 500px;
720
- text-align: center;
721
  }
722
  #toast.show { transform: translateX(-50%) translateY(0); }
723
-
724
  /* ─── Footer ─────────────────────────────────────────── */
725
  footer {
726
- padding: 12px 32px;
727
- border-top: 1px solid var(--border);
728
- display: flex;
729
- align-items: center;
730
- justify-content: space-between;
731
- font-size: 11px;
732
- color: var(--text3);
733
- font-family: 'JetBrains Mono', monospace;
734
- background: rgba(6,11,20,.8);
735
- }
736
- footer a {
737
- color: var(--text2);
738
- text-decoration: none;
739
- transition: color .15s;
740
  }
 
 
741
  footer a:hover { color: var(--accent); }
742
-
743
  /* ─── Scrollbar ──────────────────────────────────────── */
744
- ::-webkit-scrollbar { width: 6px; }
745
  ::-webkit-scrollbar-track { background: transparent; }
746
  ::-webkit-scrollbar-thumb { background: var(--border); border-radius: 3px; }
747
  ::-webkit-scrollbar-thumb:hover { background: var(--border2); }
748
-
749
- /* ─── Model color indicator ──────────────────────────── */
750
- .model-indicator {
751
- display: flex;
752
- align-items: center;
753
- gap: 6px;
754
- font-size: 11px;
755
- font-family: 'JetBrains Mono', monospace;
756
- color: var(--text2);
757
- }
758
- .model-indicator-dot {
759
- width: 8px;
760
- height: 8px;
761
- border-radius: 50%;
762
- }
763
-
764
  /* ─── Responsive ─────────────────────────────────────── */
765
- @media (max-width: 900px) {
766
- header { padding: 0 16px; }
 
 
 
 
 
 
 
767
  main { grid-template-columns: 1fr; }
768
- .input-panel { border-right: none; border-bottom: 1px solid var(--border); }
769
- .stats-row { grid-template-columns: repeat(2, 1fr); }
770
- .stat-card:nth-child(2) { border-right: none; }
771
- footer { flex-direction: column; gap: 4px; text-align: center; }
772
- }
773
-
774
- /* ─── Animations ─────────────────────────────────────── */
775
- @keyframes fadeIn {
776
- from { opacity: 0; transform: translateY(6px); }
777
- to { opacity: 1; transform: translateY(0); }
778
- }
779
- .fade-in {
780
- animation: fadeIn .25s ease forwards;
781
  }
782
  </style>
783
  </head>
784
  <body>
785
-
786
- <!-- Background -->
787
  <div class="bg-gradient"></div>
788
  <div class="dot-grid"></div>
789
-
790
  <div id="app">
791
-
792
  <!-- Header -->
793
  <header>
794
  <div class="logo">
795
  <div class="logo-hex">T</div>
796
  <span class="logo-name">TokenLens</span>
797
- <span class="logo-tag">v1.1</span>
798
  </div>
799
-
800
- <div class="model-tabs" id="model-tabs">
801
- <!-- populated by JS -->
802
- </div>
803
-
804
- <div class="custom-model-row">
805
- <label>HF model id:</label>
806
- <input class="custom-input" id="custom-model-input" type="text"
807
- placeholder="e.g. Xenova/gpt2" />
808
- <button class="btn" id="custom-model-btn">Load</button>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
809
  </div>
810
  </header>
811
-
812
  <!-- Main -->
813
- <main>
814
-
815
  <!-- Left: Input -->
816
  <div class="input-panel">
817
  <div class="panel-header">
@@ -822,204 +493,195 @@
822
  <div class="sample-btns">
823
  <button class="sample-btn" data-sample="poetry">Poetry</button>
824
  <button class="sample-btn" data-sample="code">Code</button>
825
- <button class="sample-btn" data-sample="multilingual">Multi-lingual</button>
826
  <button class="sample-btn" data-sample="numbers">Numbers</button>
827
  <button class="sample-btn" data-sample="clear">Clear</button>
828
  </div>
829
  </div>
830
  <textarea id="input-area"
831
- placeholder="Type or paste text here to see how the tokenizer splits it into tokens…
832
- &#10;&#10;Try some special characters, code snippets, emojis 🦊, or multi-lingual text (日本語, العربية) to see how different models handle them differently."></textarea>
 
833
  <div class="char-counter"><span id="char-count">0</span> characters</div>
834
  </div>
835
-
836
- <!-- Right: Output -->
837
- <div class="output-panel">
838
-
839
- <!-- Stats -->
 
 
 
840
  <div class="stats-row">
841
- <div class="stat-card" id="sc-tokens">
842
  <div class="stat-label">Tokens</div>
843
- <div class="stat-value" id="stat-tokens">—</div>
844
- <div class="stat-sub" id="stat-model-name">no model loaded</div>
845
  </div>
846
- <div class="stat-card" id="sc-chars">
847
  <div class="stat-label">Characters</div>
848
- <div class="stat-value" id="stat-chars">—</div>
849
  <div class="stat-sub">total input</div>
850
  </div>
851
- <div class="stat-card" id="sc-words">
852
  <div class="stat-label">Words</div>
853
- <div class="stat-value" id="stat-words">—</div>
854
  <div class="stat-sub">approx</div>
855
  </div>
856
- <div class="stat-card" id="sc-ratio">
857
- <div class="stat-label">Chars / Token</div>
858
- <div class="stat-value" id="stat-ratio">—</div>
859
  <div class="stat-sub">efficiency</div>
860
  </div>
861
  </div>
862
-
863
- <!-- View toggle -->
864
  <div class="view-toggle">
865
- <div class="toggle-group">
866
- <button class="toggle-btn active" data-view="text">Text View</button>
867
- <button class="toggle-btn" data-view="ids">ID Grid</button>
868
- <button class="toggle-btn" data-view="list">Token List</button>
869
  </div>
870
- <div class="model-indicator" id="model-indicator">
871
- <div class="model-indicator-dot" id="model-dot" style="background:#3d5a80"></div>
872
- <span id="model-indicator-label">no model</span>
 
 
873
  </div>
874
  </div>
875
-
876
- <!-- Token display area -->
877
- <div class="token-display" id="token-display">
878
- <div class="placeholder-msg" id="placeholder">
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
879
  <div class="placeholder-icon">⬡</div>
880
- <p>Select a model above and type something<br>to see tokenization in action</p>
881
  </div>
882
  </div>
883
-
884
- </div><!-- /output-panel -->
885
  </main>
886
-
887
  <footer>
888
  <span>TokenLens — Powered by <a href="https://github.com/xenova/transformers.js" target="_blank">Transformers.js</a> · Runs entirely in your browser</span>
889
  <span><a href="https://quickgrid.github.io/">Made by · Asif Ahmed</a></span>
890
  </footer>
891
-
892
- </div><!-- /app -->
893
-
894
  <!-- Loading Overlay -->
895
  <div id="loading-overlay">
896
  <div class="loading-spinner"></div>
897
  <div class="loading-text" id="loading-title">Loading Tokenizer</div>
898
- <div class="loading-sub" id="loading-sub">Downloading tokenizer files from Hugging Face Hub…<br>This may take a moment on first load. Files are cached in your browser.</div>
899
- <div class="loading-bar-wrap">
900
- <div class="loading-bar" id="loading-bar"></div>
901
- </div>
902
  <div class="loading-file" id="loading-file"></div>
903
  </div>
904
-
905
  <!-- Toast -->
906
  <div id="toast"></div>
907
 
908
- <!-- ─────────────────────────────────────────────────────────
909
- TokenLens Script
910
- ─────────────────────────────────────────────────────────
911
- Architecture:
912
- • Uses @xenova/transformers (Transformers.js v2) via CDN
913
- • Tokenizer files downloaded from HF Hub and cached in IndexedDB
914
- • Extends easily: add entries to MODELS registry
915
- • Supports BPE, WordPiece, SentencePiece, Unigram tokenizers
916
- ─────────────────────────────────────────────────────────── -->
917
  <script type="module">
918
-
919
  import { AutoTokenizer, env }
920
  from 'https://cdn.jsdelivr.net/npm/@xenova/transformers@2.17.2';
921
 
922
- // ── Config ────────────────────────────────────────────────
923
  env.allowLocalModels = false;
924
- // Use HF CDN for model files
925
  env.useBrowserCache = true;
926
 
927
  // ── Model Registry ─────────────────────────────────────────
928
  const MODELS = [
929
- {
930
- id: 'Xenova/gpt-4',
931
- name: 'GPT-4',
932
- org: 'OpenAI',
933
- color: '#10a37f',
934
- vocab: '100k',
935
- type: 'tiktoken cl100k',
936
- desc: 'Used by GPT-3.5 & GPT-4'
937
- },
938
- {
939
- id: 'Xenova/mistral-tokenizer-v1',
940
- name: 'Mistral',
941
- org: 'Mistral AI',
942
- color: '#ff7722',
943
- vocab: '32k',
944
- type: 'SP-BPE',
945
- desc: 'Mistral 7B v0.1 tokenizer'
946
- },
947
- {
948
- id: 'Xenova/claude-tokenizer',
949
- name: 'Claude',
950
- org: 'Anthropic',
951
- color: '#cc785c',
952
- vocab: '~100k',
953
- type: 'BPE',
954
- desc: "Anthropic Claude's tokenizer"
955
- },
956
  ];
957
 
958
- // ── Token Color Palette ───────────────────────────────────
959
- const PALETTE = [
960
- { text: '#ff8080', bg: 'rgba(255,128,128,.18)', border: 'rgba(255,128,128,.35)' },
961
- { text: '#ffb84d', bg: 'rgba(255,184, 77,.18)', border: 'rgba(255,184, 77,.35)' },
962
- { text: '#ffe066', bg: 'rgba(255,224,102,.18)', border: 'rgba(255,224,102,.35)' },
963
- { text: '#7aed91', bg: 'rgba(122,237,145,.18)', border: 'rgba(122,237,145,.35)' },
964
- { text: '#4ddfc0', bg: 'rgba( 77,223,192,.18)', border: 'rgba( 77,223,192,.35)' },
965
- { text: '#56c8f5', bg: 'rgba( 86,200,245,.18)', border: 'rgba( 86,200,245,.35)' },
966
- { text: '#748ef8', bg: 'rgba(116,142,248,.18)', border: 'rgba(116,142,248,.35)' },
967
- { text: '#c484f8', bg: 'rgba(196,132,248,.18)', border: 'rgba(196,132,248,.35)' },
968
- { text: '#f57cd4', bg: 'rgba(245,124,212,.18)', border: 'rgba(245,124,212,.35)' },
969
- { text: '#fa8072', bg: 'rgba(250,128,114,.18)', border: 'rgba(250,128,114,.35)' },
970
- { text: '#8be08b', bg: 'rgba(139,224,139,.18)', border: 'rgba(139,224,139,.35)' },
971
- { text: '#f0c040', bg: 'rgba(240,192, 64,.18)', border: 'rgba(240,192, 64,.35)' },
972
- { text: '#60d4e0', bg: 'rgba( 96,212,224,.18)', border: 'rgba( 96,212,224,.35)' },
973
- { text: '#e89060', bg: 'rgba(232,144, 96,.18)', border: 'rgba(232,144, 96,.35)' },
974
  ];
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
975
 
976
  // ── Sample texts ───────────────────────────────────────────
977
  const SAMPLES = {
978
- poetry: `Two roads diverged in a yellow wood,
979
- And sorry I could not travel both
980
- And be one traveler, long I stood
981
- And looked down one as far as I could
982
- To where it bent in the undergrowth;
983
-
984
- — Robert Frost, "The Road Not Taken"`,
985
-
986
- code: `async function fetchData(url, retries = 3) {
987
- for (let i = 0; i < retries; i++) {
988
- try {
989
- const res = await fetch(url);
990
- if (!res.ok) throw new Error(\`HTTP \${res.status}\`);
991
- return await res.json();
992
- } catch (e) {
993
- if (i === retries - 1) throw e;
994
- await new Promise(r => setTimeout(r, 1000 * 2 ** i));
995
- }
996
- }
997
- }`,
998
-
999
- multilingual: `English: The quick brown fox jumps over the lazy dog.
1000
- 日本語: 吾輩は猫である。名前はまだない。
1001
- 中文: 春眠不觉晓,处处闻啼鸟。
1002
- العربية: اللغة العربية جميلة ومعبرة.
1003
- Ελληνικά: Η γνώση είναι δύναμη.
1004
- Emoji: 🌍 🦊 ⚡ 🎯 🧬 🤖 🦋`,
1005
-
1006
- numbers: `π ≈ 3.14159265358979323846
1007
- e ≈ 2.71828182845904523536
1008
- φ ≈ 1.61803398874989484820
1009
- 1,000,000 × $42.99 = $42,990,000.00
1010
- 2024-01-15T08:30:00.000Z
1011
- IPv4: 192.168.1.1 | IPv6: ::1`,
1012
-
1013
  clear: ''
1014
  };
1015
 
1016
  // ── State ──────────────────────────────────────────────────
1017
- let activeTokenizer = null;
1018
- let activeModel = null;
1019
- let tokenizerCache = {}; // modelId tokenizer
1020
- let currentView = 'text';
1021
- let showSpecial = false;
1022
- let debounceTimer = null;
 
1023
 
1024
  // ── DOM References ─────────────────────────────────────────
1025
  const $overlay = document.getElementById('loading-overlay');
@@ -1027,24 +689,14 @@ const $loadTitle = document.getElementById('loading-title');
1027
  const $loadSub = document.getElementById('loading-sub');
1028
  const $loadBar = document.getElementById('loading-bar');
1029
  const $loadFile = document.getElementById('loading-file');
1030
- const $modelTabs = document.getElementById('model-tabs');
1031
  const $input = document.getElementById('input-area');
1032
  const $charCount = document.getElementById('char-count');
1033
- const $display = document.getElementById('token-display');
1034
- const $placeholder = document.getElementById('placeholder');
1035
- const $stTokens = document.getElementById('stat-tokens');
1036
- const $stChars = document.getElementById('stat-chars');
1037
- const $stWords = document.getElementById('stat-words');
1038
- const $stRatio = document.getElementById('stat-ratio');
1039
- const $stModelName = document.getElementById('stat-model-name');
1040
- const $modelDot = document.getElementById('model-dot');
1041
- const $modelLabel = document.getElementById('model-indicator-label');
1042
  const $toast = document.getElementById('toast');
1043
- const $customInput = document.getElementById('custom-model-input');
1044
- const $customBtn = document.getElementById('custom-model-btn');
 
1045
 
1046
  // ── Utilities ──────────────────────────────────────────────
1047
-
1048
  function showOverlay(title, sub) {
1049
  $loadTitle.textContent = title;
1050
  $loadSub.textContent = sub;
@@ -1052,10 +704,7 @@ function showOverlay(title, sub) {
1052
  $loadFile.textContent = '';
1053
  $overlay.classList.remove('hidden');
1054
  }
1055
-
1056
- function hideOverlay() {
1057
- $overlay.classList.add('hidden');
1058
- }
1059
 
1060
  function showToast(msg, duration = 5000) {
1061
  $toast.textContent = msg;
@@ -1063,25 +712,31 @@ function showToast(msg, duration = 5000) {
1063
  setTimeout(() => $toast.classList.remove('show'), duration);
1064
  }
1065
 
1066
- function setStats(tokens, text) {
1067
  const chars = text.length;
1068
  const words = text.trim() ? text.trim().split(/\s+/).length : 0;
1069
  const ratio = tokens > 0 && chars > 0 ? (chars / tokens).toFixed(2) : '—';
1070
-
1071
- $stTokens.textContent = tokens > 0 ? tokens.toLocaleString() : '—';
1072
- $stChars.textContent = chars > 0 ? chars.toLocaleString() : '—';
1073
- $stWords.textContent = words > 0 ? words.toLocaleString() : '—';
1074
- $stRatio.textContent = ratio;
1075
-
1076
- // Pulse animation
1077
- ['sc-tokens','sc-chars','sc-words','sc-ratio'].forEach(id => {
1078
- const el = document.getElementById(id);
1079
- el.classList.remove('highlight');
1080
- void el.offsetWidth;
1081
- el.classList.add('highlight');
1082
  });
1083
  }
1084
 
 
 
 
 
 
 
 
 
 
 
 
1085
  // ── Decode raw token string for display ───────────────────
1086
  function decodeTokenString(raw) {
1087
  if (!raw) return '';
@@ -1096,192 +751,149 @@ function decodeTokenString(raw) {
1096
  return s;
1097
  }
1098
 
1099
- // ── Tokenize ───────────────────────────────────────────────
1100
- async function tokenize(text) {
1101
- if (!activeTokenizer || !text.trim()) {
1102
- $display.innerHTML = '';
1103
- $display.appendChild($placeholder);
1104
- $placeholder.style.display = 'flex';
1105
- setStats(0, text);
 
 
 
 
1106
  return;
1107
  }
1108
-
1109
  try {
1110
- $placeholder.style.display = 'none';
1111
-
1112
- const encoded = await activeTokenizer(text, {
1113
- add_special_tokens: showSpecial,
1114
- return_offsets_mapping: false,
1115
- });
1116
-
1117
  const ids = Array.from(encoded.input_ids.data);
1118
-
1119
  let rawTokens;
1120
- try {
1121
- rawTokens = activeTokenizer.model.convert_ids_to_tokens(ids);
1122
- } catch {
1123
- rawTokens = await Promise.all(
1124
- ids.map(id => activeTokenizer.decode([id], { skip_special_tokens: false }))
1125
- );
1126
- }
1127
-
1128
  const tokens = ids.map((id, i) => ({
1129
- id,
1130
- raw: rawTokens[i] || '',
1131
- display: decodeTokenString(rawTokens[i] || ''),
1132
  }));
1133
-
1134
- setStats(tokens.length, text);
1135
- renderView(tokens);
1136
-
1137
  } catch (err) {
1138
  console.error('Tokenization error:', err);
1139
- showToast('Tokenization error: ' + err.message);
1140
  }
1141
  }
1142
 
1143
  // ── Render Views ───────────────────────────────────────────
1144
-
1145
- function renderView(tokens) {
1146
- if (currentView === 'text') renderTextView(tokens);
1147
- else if (currentView === 'ids') renderIdView(tokens);
1148
- else if (currentView === 'list') renderListView(tokens);
1149
  }
1150
 
1151
- function renderTextView(tokens) {
 
 
1152
  const container = document.createElement('div');
1153
- container.className = 'token-text-view fade-in';
1154
-
1155
  tokens.forEach((tok, i) => {
1156
  const c = PALETTE[i % PALETTE.length];
1157
  const span = document.createElement('span');
1158
  span.className = 'tok';
1159
- span.style.background = c.bg;
1160
- span.style.color = c.text;
1161
- span.style.borderBottom = `2px solid ${c.border}`;
1162
-
1163
  const disp = tok.display;
1164
- if (disp === ' ') {
1165
- span.innerHTML = '&nbsp;';
1166
- } else if (disp === '\n') {
1167
- span.innerHTML = '↵<br>';
1168
- } else if (disp === '\t') {
1169
- span.innerHTML = '→&nbsp;&nbsp;&nbsp;';
1170
- } else {
1171
- span.textContent = disp;
1172
- }
1173
-
1174
  const tip = document.createElement('div');
1175
  tip.className = 'tok-tooltip';
1176
-
1177
- const rawEsc = tok.raw
1178
- .replace(/&/g,'&amp;')
1179
- .replace(/</g,'&lt;')
1180
- .replace(/>/g,'&gt;');
1181
-
1182
- tip.innerHTML =
1183
- `<span class="tok-tooltip-id">#${tok.id}</span> · ` +
1184
- `<span class="tok-tooltip-text">${rawEsc || '(empty)'}</span>`;
1185
  span.appendChild(tip);
1186
-
1187
  container.appendChild(span);
1188
  });
1189
-
1190
- $display.innerHTML = '';
 
1191
  $display.appendChild(container);
1192
  }
1193
 
1194
- function renderIdView(tokens) {
 
 
1195
  const container = document.createElement('div');
1196
- container.className = 'token-id-view fade-in';
1197
-
1198
  tokens.forEach((tok, i) => {
1199
  const c = PALETTE[i % PALETTE.length];
1200
  const card = document.createElement('div');
1201
- card.className = 'tok-id-card';
1202
- card.style.background = c.bg;
1203
- card.style.borderColor = c.border;
1204
  card.title = `Raw: ${tok.raw}`;
1205
-
1206
  const top = document.createElement('div');
1207
- top.className = 'tok-id-top';
1208
- top.style.color = c.text;
1209
- top.textContent = tok.id;
1210
-
1211
  const bot = document.createElement('div');
1212
  bot.className = 'tok-id-bottom';
1213
- const label = tok.display.slice(0, 8).replace(/\n/g,'↵').replace(/\t/g,'→');
1214
- bot.textContent = label || '…';
1215
-
1216
- card.appendChild(top);
1217
- card.appendChild(bot);
1218
  container.appendChild(card);
1219
  });
1220
-
1221
- $display.innerHTML = '';
 
1222
  $display.appendChild(container);
1223
  }
1224
 
1225
- function renderListView(tokens) {
 
 
1226
  const container = document.createElement('div');
1227
- container.className = 'token-split-view fade-in';
1228
-
1229
  tokens.forEach((tok, i) => {
1230
  const c = PALETTE[i % PALETTE.length];
1231
  const row = document.createElement('div');
1232
  row.className = 'tok-split-row';
1233
  row.style.background = c.bg;
1234
  row.style.borderColor = c.border;
1235
-
1236
- const idx = document.createElement('div');
1237
- idx.className = 'tok-split-idx';
1238
- idx.textContent = i;
1239
-
1240
- const text = document.createElement('div');
1241
- text.className = 'tok-split-text';
1242
- text.style.color = c.text;
1243
- const disp = tok.display.replace(/\n/g,'↵').replace(/\t/g,'→') || '(empty)';
1244
- text.textContent = disp;
1245
-
1246
- const id = document.createElement('div');
1247
- id.className = 'tok-split-id';
1248
- id.textContent = tok.id;
1249
-
1250
- row.appendChild(idx);
1251
- row.appendChild(text);
1252
- row.appendChild(id);
1253
  container.appendChild(row);
1254
  });
1255
-
1256
- $display.innerHTML = '';
 
1257
  $display.appendChild(container);
1258
  }
1259
 
1260
  // ── Load Tokenizer ─────────────────────────────────────────
1261
-
1262
- async function loadModel(modelId) {
1263
  if (tokenizerCache[modelId]) {
1264
- activeTokenizer = tokenizerCache[modelId];
1265
- updateModelIndicator(modelId);
 
1266
  await runTokenize();
1267
  return;
1268
  }
1269
-
1270
  const displayName = modelId.split('/').pop();
 
1271
  showOverlay(
1272
- `Loading ${displayName}`,
1273
- `Fetching tokenizer.json and tokenizer_config.json from Hugging Face Hub.\nFiles are cached in IndexedDB after first download.`
1274
  );
1275
-
1276
  let lastProgress = 0;
1277
-
1278
  try {
1279
  const tokenizer = await AutoTokenizer.from_pretrained(modelId, {
1280
  progress_callback: (info) => {
1281
  if (info.status === 'downloading') {
1282
- const pct = info.total
1283
- ? Math.round((info.loaded / info.total) * 100)
1284
- : lastProgress;
1285
  $loadBar.style.width = pct + '%';
1286
  $loadFile.textContent = info.file || '';
1287
  lastProgress = pct;
@@ -1290,121 +902,160 @@ async function loadModel(modelId) {
1290
  }
1291
  }
1292
  });
1293
-
1294
  tokenizerCache[modelId] = tokenizer;
1295
- activeTokenizer = tokenizer;
1296
- activeModel = modelId;
1297
-
1298
- updateModelIndicator(modelId);
1299
  hideOverlay();
1300
  await runTokenize();
1301
-
1302
  } catch (err) {
1303
  hideOverlay();
1304
  console.error('Failed to load tokenizer:', err);
1305
- showToast(`Failed to load "${modelId}": ${err.message}. Check the model ID and try again.`, 8000);
1306
  }
1307
  }
1308
 
1309
- function updateModelIndicator(modelId) {
1310
- const preset = MODELS.find(m => m.id === modelId);
1311
- const color = preset ? preset.color : '#7899c0';
1312
- const name = modelId.split('/').pop();
1313
- $modelDot.style.background = color;
1314
- $modelDot.style.boxShadow = `0 0 6px ${color}`;
1315
- $modelLabel.textContent = name;
1316
- $stModelName.textContent = preset ? `${preset.org} · ${preset.type} · ${preset.vocab} vocab` : modelId;
1317
- }
1318
-
1319
- // ── Build Model Tabs ───────────────────────────────────────
1320
-
1321
- function buildTabs() {
1322
- $modelTabs.innerHTML = '';
1323
- MODELS.forEach(m => {
1324
- const tab = document.createElement('div');
1325
- tab.className = 'model-tab';
1326
- tab.dataset.id = m.id;
1327
- tab.title = m.desc;
1328
- tab.innerHTML = `
1329
- <div class="model-tab-name">${m.name}</div>
1330
- <div class="model-tab-org">
1331
- <span class="model-org-dot" style="background:${m.color}"></span>${m.org}
1332
- </div>
1333
- <div class="model-tab-vocab">${m.type} · ${m.vocab} vocab</div>
1334
- `;
1335
- tab.addEventListener('click', () => selectTab(m.id));
1336
- $modelTabs.appendChild(tab);
1337
  });
1338
  }
1339
 
1340
- function selectTab(modelId) {
1341
- document.querySelectorAll('.model-tab').forEach(t => {
1342
- t.classList.toggle('active', t.dataset.id === modelId);
 
 
 
 
 
 
1343
  });
1344
- loadModel(modelId);
1345
- }
1346
 
1347
- // ── View Toggle ────────────────────────────────────────────
 
 
 
 
 
1348
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1349
  document.querySelectorAll('.toggle-btn').forEach(btn => {
1350
  btn.addEventListener('click', () => {
1351
- document.querySelectorAll('.toggle-btn').forEach(b => b.classList.remove('active'));
 
 
1352
  btn.classList.add('active');
1353
- currentView = btn.dataset.view;
1354
  runTokenize();
1355
  });
1356
  });
1357
 
1358
  // ── Input Handling ─────────────────────────────────────────
1359
-
1360
  async function runTokenize() {
1361
  const text = $input.value;
1362
  $charCount.textContent = text.length;
1363
- await tokenize(text);
 
 
 
1364
  }
1365
 
1366
- $input.addEventListener('input', () => {
1367
  $charCount.textContent = $input.value.length;
1368
  clearTimeout(debounceTimer);
1369
  debounceTimer = setTimeout(runTokenize, 280);
1370
  });
1371
 
1372
  // ── Sample Buttons ─────────────────────────────────────────
1373
-
1374
  document.querySelectorAll('.sample-btn').forEach(btn => {
1375
  btn.addEventListener('click', () => {
1376
- const key = btn.dataset.sample;
1377
- $input.value = SAMPLES[key] ?? '';
1378
  $input.focus();
1379
  runTokenize();
1380
  });
1381
  });
1382
 
1383
- // ── Custom Model ───────────────────────────────────────────
1384
-
1385
- async function loadCustomModel() {
1386
- const id = $customInput.value.trim();
1387
- if (!id) { showToast('Please enter a model ID'); return; }
 
 
 
 
 
 
 
 
 
 
 
 
1388
 
1389
- document.querySelectorAll('.model-tab').forEach(t => t.classList.remove('active'));
1390
- activeModel = id;
1391
- await loadModel(id);
 
 
 
 
 
 
 
 
 
 
 
1392
  }
1393
 
1394
- $customBtn.addEventListener('click', loadCustomModel);
1395
- $customInput.addEventListener('keydown', e => {
1396
- if (e.key === 'Enter') loadCustomModel();
1397
  });
1398
 
1399
  // ── Init ───────────────────────────────────────────────────
 
 
 
1400
 
1401
- buildTabs();
1402
- $overlay.classList.add('hidden');
1403
-
1404
- $input.value = '';
1405
-
1406
- selectTab(MODELS[0].id);
1407
 
 
 
1408
  </script>
1409
  </body>
1410
- </html>
 
1
  <!DOCTYPE html>
2
+ <html lang="en" data-theme="dark">
3
  <head>
4
  <meta charset="UTF-8" />
5
  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
 
7
  <meta name="description" content="Visualize how large language models tokenize text. Powered by Transformers.js, runs entirely in your browser." />
8
  <link rel="preconnect" href="https://fonts.googleapis.com" />
9
  <link href="https://fonts.googleapis.com/css2?family=Bricolage+Grotesque:opsz,wght@12..96,300;12..96,400;12..96,500;12..96,600;12..96,700;12..96,800&family=JetBrains+Mono:wght@300;400;500;700&family=DM+Sans:wght@300;400;500&display=swap" rel="stylesheet" />
 
10
  <style>
11
  /* ─── Design Tokens ─────────────────────────────────── */
12
  :root {
 
25
  --green: #34d89a;
26
  --amber: #f5a623;
27
  --red: #f55577;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
28
  }
29
+ [data-theme="light"] {
30
+ --bg: #eef1f8;
31
+ --bg2: #e2e7f2;
32
+ --bg3: #d3dae8;
33
+ --bg4: #c2cce0;
34
+ --border: #b8c4d8;
35
+ --border2: #a0aec8;
36
+ --glow: #c0d0e8;
37
+ --text: #1a2236;
38
+ --text2: #5a6888;
39
+ --text3: #8898b4;
40
+ --accent: #2878e0;
41
+ --accent2: #6838d8;
42
+ --green: #18a060;
43
+ --amber: #c88010;
44
+ --red: #d83858;
45
+ }
46
  /* ─── Reset ─────────────────────────────────────────── */
47
  *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
48
+ html { height: 100%; overflow: hidden; }
 
 
49
  body {
50
  background: var(--bg);
51
  color: var(--text);
52
  font-family: 'DM Sans', sans-serif;
53
+ height: 100%;
54
+ overflow: hidden;
 
 
55
  }
 
56
  /* ─── Background FX ─────────────────────────────────── */
 
 
 
 
 
 
57
  .bg-gradient {
58
+ position: fixed; inset: 0; pointer-events: none; z-index: 0;
 
 
 
59
  background:
60
  radial-gradient(ellipse 80% 50% at 20% 10%, rgba(77,158,245,.06) 0%, transparent 70%),
61
  radial-gradient(ellipse 60% 40% at 80% 90%, rgba(139,106,245,.05) 0%, transparent 60%),
62
  radial-gradient(ellipse 40% 30% at 60% 50%, rgba(52,216,154,.03) 0%, transparent 60%);
63
  }
64
+ [data-theme="light"] .bg-gradient {
65
+ background:
66
+ radial-gradient(ellipse 80% 50% at 20% 10%, rgba(40,120,224,.05) 0%, transparent 70%),
67
+ radial-gradient(ellipse 60% 40% at 80% 90%, rgba(104,56,216,.04) 0%, transparent 60%),
68
+ radial-gradient(ellipse 40% 30% at 60% 50%, rgba(24,160,96,.02) 0%, transparent 60%);
69
+ }
70
  .dot-grid {
71
+ position: fixed; inset: 0; pointer-events: none; z-index: 0;
 
 
 
72
  background-image: radial-gradient(circle, rgba(77,158,245,.12) 1px, transparent 1px);
73
  background-size: 36px 36px;
74
  mask-image: radial-gradient(ellipse 100% 100% at 50% 50%, black 30%, transparent 80%);
75
  }
76
+ [data-theme="light"] .dot-grid {
77
+ background-image: radial-gradient(circle, rgba(40,120,224,.06) 1px, transparent 1px);
78
+ }
79
  /* ─── Layout ─────────────────────────────────────────── */
80
  #app {
81
+ position: relative; z-index: 1;
82
+ display: flex; flex-direction: column;
83
+ height: 100vh; overflow: hidden;
 
 
84
  }
 
85
  /* ─── Header ─────────────────────────────────────────── */
86
  header {
87
+ display: flex; align-items: center;
88
+ padding: 0 20px; height: 56px;
 
 
 
89
  border-bottom: 1px solid var(--border);
90
  background: rgba(6,11,20,.85);
91
  backdrop-filter: blur(20px);
92
+ flex-shrink: 0; z-index: 100; gap: 12px;
 
 
 
93
  }
94
+ [data-theme="light"] header { background: rgba(238,241,248,.92); }
95
  .logo {
96
+ display: flex; align-items: center; gap: 8px;
97
+ text-decoration: none; color: var(--text); flex-shrink: 0;
 
 
 
 
98
  }
99
  .logo-hex {
100
+ width: 30px; height: 30px;
 
101
  background: linear-gradient(135deg, var(--accent), var(--accent2));
102
  clip-path: polygon(50% 0%, 93% 25%, 93% 75%, 50% 100%, 7% 75%, 7% 25%);
103
+ display: flex; align-items: center; justify-content: center;
104
+ font-size: 12px; font-family: 'JetBrains Mono', monospace; font-weight: 700; color: white;
 
 
 
 
 
105
  }
106
  .logo-name {
107
  font-family: 'Bricolage Grotesque', sans-serif;
108
+ font-size: 17px; font-weight: 700; letter-spacing: -0.5px;
 
 
109
  background: linear-gradient(135deg, #dce8f8 40%, var(--accent));
110
+ -webkit-background-clip: text; -webkit-text-fill-color: transparent; background-clip: text;
 
 
111
  }
112
+ [data-theme="light"] .logo-name {
113
+ background: linear-gradient(135deg, #1a2236 40%, var(--accent));
114
+ -webkit-background-clip: text; -webkit-text-fill-color: transparent; background-clip: text;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
115
  }
116
+ .logo-tag {
117
+ font-size: 9px; font-family: 'JetBrains Mono', monospace; color: var(--text3);
118
+ background: var(--bg3); border: 1px solid var(--border);
119
+ padding: 1px 5px; border-radius: 4px; letter-spacing: .5px;
120
+ }
121
+ .header-divider {
122
+ width: 1px; height: 28px; background: var(--border); flex-shrink: 0;
123
+ }
124
+ /* ─── Search Bar Groups ─────────────────────────────── */
125
+ .header-controls {
126
+ display: flex; align-items: center; gap: 10px; flex: 1; min-width: 0;
127
+ }
128
+ .searchbar-group {
129
+ position: relative; display: flex; align-items: center; gap: 3px; flex: 1; min-width: 0;
130
+ }
131
+ .searchbar-label {
132
+ font-family: 'JetBrains Mono', monospace; font-size: 11px; font-weight: 700;
133
+ width: 22px; height: 22px; border-radius: 5px; display: flex; align-items: center;
134
+ justify-content: center; flex-shrink: 0; border: 1px solid var(--border);
135
+ }
136
+ .searchbar-label.label-a { background: rgba(77,158,245,.15); color: var(--accent); border-color: rgba(77,158,245,.3); }
137
+ .searchbar-label.label-b { background: rgba(139,106,245,.15); color: var(--accent2); border-color: rgba(139,106,245,.3); }
138
+ .searchbar-input {
139
+ flex: 1; min-width: 0; background: var(--bg2); border: 1px solid var(--border);
140
+ border-radius: 6px; color: var(--text); font-family: 'JetBrains Mono', monospace;
141
+ font-size: 11px; padding: 5px 8px; outline: none; transition: border-color .2s;
142
+ }
143
+ .searchbar-input:focus { border-color: var(--accent); }
144
+ .searchbar-input::placeholder { color: var(--text3); }
145
+ .searchbar-dropdown-btn {
146
+ width: 24px; height: 24px; border-radius: 5px; border: 1px solid var(--border);
147
+ background: var(--bg2); color: var(--text2); cursor: pointer; font-size: 10px;
148
+ display: flex; align-items: center; justify-content: center; flex-shrink: 0;
149
+ transition: all .15s;
150
  }
151
+ .searchbar-dropdown-btn:hover { border-color: var(--border2); color: var(--text); }
152
+ .searchbar-load-btn {
153
+ padding: 4px 10px; border-radius: 5px; border: 1px solid var(--border2);
154
+ background: linear-gradient(135deg, rgba(77,158,245,.12), rgba(139,106,245,.12));
155
+ color: var(--accent); font-family: 'DM Sans', sans-serif; font-size: 11px;
156
+ font-weight: 500; cursor: pointer; transition: all .15s; white-space: nowrap; flex-shrink: 0;
157
  }
158
+ .searchbar-load-btn:hover {
159
+ background: linear-gradient(135deg, rgba(77,158,245,.22), rgba(139,106,245,.22));
160
  border-color: var(--accent);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
161
  }
162
+ /* ─── Dropdown Menu ─────────────────────────────────── */
163
+ .dropdown-menu {
164
+ position: absolute; top: calc(100% + 6px); left: 22px; right: 0;
165
+ min-width: 280px; max-height: 320px; overflow-y: auto;
166
+ background: var(--bg3); border: 1px solid var(--border2); border-radius: 8px;
167
+ z-index: 200; display: none; padding: 4px;
168
+ box-shadow: 0 8px 32px rgba(0,0,0,.4);
169
+ }
170
+ [data-theme="light"] .dropdown-menu { box-shadow: 0 8px 32px rgba(0,0,0,.12); }
171
+ .dropdown-menu.open { display: block; }
172
+ .dropdown-item {
173
+ padding: 7px 10px; cursor: pointer; border-radius: 5px;
174
+ display: flex; align-items: center; gap: 8px; font-size: 12px;
175
+ transition: background .12s;
176
+ }
177
+ .dropdown-item:hover { background: var(--bg4); }
178
+ .dropdown-item-dot {
179
+ width: 6px; height: 6px; border-radius: 50%; flex-shrink: 0;
180
+ }
181
+ .dropdown-item-name { font-weight: 600; color: var(--text); font-family: 'Bricolage Grotesque', sans-serif; }
182
+ .dropdown-item-detail { font-size: 10px; color: var(--text3); font-family: 'JetBrains Mono', monospace; margin-left: auto; white-space: nowrap; }
183
+ /* ─── Icon Toggle Buttons ────────────────────────────── */
184
+ .icon-toggle-btn {
185
+ width: 24px; height: 24px; border-radius: 7px; border: 1px solid var(--border);
186
+ background: var(--bg2); color: var(--text2); cursor: pointer; flex-shrink: 0;
187
+ display: flex; align-items: center; justify-content: center; transition: all .15s;
188
+ }
189
+ .icon-toggle-btn:hover { border-color: var(--border2); color: var(--text); }
190
+ .icon-toggle-btn.active { border-color: var(--accent); color: var(--accent); background: rgba(77,158,245,.1); }
191
+ .icon-toggle-btn svg { width: 16px; height: 16px; }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
192
  /* ─── Main Split ─────────────────────────────────────── */
193
  main {
194
+ flex: 1; min-height: 0; overflow: hidden;
195
+ display: grid; grid-template-columns: 1fr 1fr 1fr; gap: 0;
 
 
 
196
  }
197
+ main.single-panel { grid-template-columns: 1fr 2fr; }
198
  /* ─── Left Panel (Input) ─────────────────────────────── */
199
  .input-panel {
200
  border-right: 1px solid var(--border);
201
+ display: flex; flex-direction: column; overflow: hidden;
 
 
202
  }
203
  .panel-header {
204
+ padding: 12px 16px 10px; border-bottom: 1px solid var(--border);
205
+ display: flex; align-items: center; justify-content: space-between; flex-shrink: 0;
 
 
 
206
  }
207
  .panel-title {
208
+ font-family: 'Bricolage Grotesque', sans-serif; font-size: 13px; font-weight: 600;
209
+ color: var(--text2); letter-spacing: .3px; display: flex; align-items: center; gap: 6px;
 
 
 
 
 
 
210
  }
211
  .panel-title-icon {
212
+ width: 18px; height: 18px; background: var(--bg4); border: 1px solid var(--border);
213
+ border-radius: 4px; display: flex; align-items: center; justify-content: center; font-size: 10px;
 
 
 
 
 
 
 
 
 
 
 
 
214
  }
215
+ .sample-btns { display: flex; gap: 4px; flex-wrap: wrap; }
216
  .sample-btn {
217
+ font-size: 10px; padding: 3px 8px; border-radius: 5px;
218
+ border: 1px solid var(--border); background: var(--bg2); color: var(--text2);
219
+ cursor: pointer; font-family: 'DM Sans', sans-serif; transition: all .15s;
 
 
 
 
 
 
220
  }
221
+ .sample-btn:hover { border-color: var(--border2); color: var(--text); }
 
 
 
 
222
  #input-area {
223
+ flex: 1; width: 100%; background: transparent; border: none; outline: none;
224
+ resize: none; color: var(--text); font-family: 'DM Sans', sans-serif;
225
+ font-size: 14px; line-height: 1.7; padding: 14px 16px; min-height: 0; overflow-y: auto;
 
 
 
 
 
 
 
 
 
226
  }
227
  #input-area::placeholder { color: var(--text3); }
 
228
  .char-counter {
229
+ padding: 6px 16px; border-top: 1px solid var(--border); flex-shrink: 0;
230
+ font-size: 10px; font-family: 'JetBrains Mono', monospace; color: var(--text3); text-align: right;
 
 
 
 
231
  }
232
+ /* ─── Output Panel ───────────────────────────────────── */
 
233
  .output-panel {
234
+ display: flex; flex-direction: column; overflow: hidden; min-height: 0;
 
 
235
  }
236
+ .output-panel + .output-panel { border-left: 1px solid var(--border); }
237
+ .output-panel-header {
238
+ padding: 10px 14px; border-bottom: 1px solid var(--border);
239
+ display: flex; align-items: center; justify-content: space-between; flex-shrink: 0; gap: 8px;
240
+ }
241
+ .model-indicator {
242
+ display: flex; align-items: center; gap: 5px; font-size: 11px;
243
+ font-family: 'JetBrains Mono', monospace; color: var(--text2); min-width: 0; overflow: hidden;
244
+ }
245
+ .model-indicator-dot { width: 7px; height: 7px; border-radius: 50%; flex-shrink: 0; }
246
+ .model-indicator-name { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
247
  /* Stats row */
248
  .stats-row {
249
+ display: grid; grid-template-columns: 1fr 1fr;
250
+ border-bottom: 1px solid var(--border); flex-shrink: 0;
 
251
  }
252
  .stat-card {
253
+ padding: 10px 14px; border-right: 1px solid var(--border);
254
+ border-bottom: 1px solid var(--border); position: relative; overflow: hidden;
 
 
255
  }
256
+ .stat-card:nth-child(2n) { border-right: none; }
257
+ .stat-card:nth-child(3), .stat-card:nth-child(4) { border-bottom: none; }
258
  .stat-card::after {
259
+ content: ''; position: absolute; bottom: 0; left: 0; right: 0; height: 2px;
 
 
 
 
 
260
  background: linear-gradient(90deg, transparent, var(--accent), transparent);
261
+ opacity: 0; transition: opacity .3s;
 
262
  }
263
  .stat-card.highlight::after { opacity: 1; }
264
  .stat-label {
265
+ font-size: 9px; font-family: 'JetBrains Mono', monospace; color: var(--text3);
266
+ text-transform: uppercase; letter-spacing: 1px; margin-bottom: 4px;
 
 
 
 
267
  }
268
  .stat-value {
269
+ font-family: 'Bricolage Grotesque', sans-serif; font-size: 20px; font-weight: 700;
270
+ color: var(--text); line-height: 1; transition: all .3s;
 
 
 
 
271
  }
272
  .stat-card:nth-child(1) .stat-value { color: var(--accent); }
273
  .stat-card:nth-child(2) .stat-value { color: var(--green); }
274
  .stat-card:nth-child(3) .stat-value { color: var(--amber); }
275
  .stat-card:nth-child(4) .stat-value { color: var(--accent2); }
276
  .stat-sub {
277
+ font-size: 9px; color: var(--text3); font-family: 'JetBrains Mono', monospace; margin-top: 2px;
 
 
 
278
  }
 
279
  /* View toggle */
280
  .view-toggle {
281
+ display: flex; padding: 8px 14px; border-bottom: 1px solid var(--border);
282
+ gap: 4px; align-items: center; justify-content: space-between; flex-shrink: 0;
 
 
 
 
283
  }
284
  .toggle-group {
285
+ display: flex; gap: 2px; background: var(--bg2); border: 1px solid var(--border);
286
+ border-radius: 6px; padding: 2px;
 
 
 
 
287
  }
288
  .toggle-btn {
289
+ padding: 3px 10px; border-radius: 4px; border: none; background: transparent;
290
+ color: var(--text2); font-family: 'DM Sans', sans-serif; font-size: 11px;
291
+ font-weight: 500; cursor: pointer; transition: all .15s;
 
 
 
 
 
 
 
292
  }
293
+ .toggle-btn.active { background: var(--bg4); color: var(--text); box-shadow: 0 1px 4px rgba(0,0,0,.3); }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
294
  /* Token Display */
295
  .token-display {
296
+ flex: 1; overflow-y: auto; padding: 14px; min-height: 0;
297
+ scrollbar-width: thin; scrollbar-color: var(--border) transparent;
 
 
 
298
  }
 
299
  .placeholder-msg {
300
+ display: flex; flex-direction: column; align-items: center; justify-content: center;
301
+ height: 100%; min-height: 120px; gap: 12px; color: var(--text3);
 
 
 
 
 
 
 
 
 
302
  }
303
+ .placeholder-icon { font-size: 32px; filter: grayscale(1) opacity(.3); }
304
  .placeholder-msg p {
305
+ font-family: 'JetBrains Mono', monospace; font-size: 11px; text-align: center; line-height: 1.6;
 
 
306
  }
 
307
  /* ─── Token Visualization Views ───────────────────────── */
 
 
308
  .token-text-view {
309
+ font-family: 'JetBrains Mono', monospace; font-size: 13px;
310
+ line-height: 2.2; word-break: break-all;
 
 
311
  }
312
  .tok {
313
+ display: inline; border-radius: 3px; padding: 1px 0;
314
+ cursor: default; transition: filter .15s; position: relative;
 
 
 
 
315
  }
316
  .tok:hover { filter: brightness(1.3); }
317
  .tok-tooltip {
318
+ display: none; position: absolute; bottom: 110%; left: 50%;
319
+ transform: translateX(-50%); background: var(--bg4); border: 1px solid var(--border2);
320
+ border-radius: 5px; padding: 4px 7px; font-size: 10px; white-space: nowrap;
321
+ z-index: 50; pointer-events: none; box-shadow: 0 4px 20px rgba(0,0,0,.5);
 
 
 
 
 
 
 
 
 
 
322
  }
323
+ [data-theme="light"] .tok-tooltip { box-shadow: 0 4px 16px rgba(0,0,0,.12); }
324
  .tok:hover .tok-tooltip { display: block; }
325
  .tok-tooltip-id { color: var(--accent); font-weight: 700; }
326
  .tok-tooltip-text { color: var(--text2); }
327
  .tok-space::before { content: '·'; opacity: .3; }
328
  .tok-newline::before { content: '↵'; opacity: .5; }
329
+ /* ID VIEW */
330
+ .token-id-view { display: flex; flex-wrap: wrap; gap: 5px; }
 
 
 
 
 
331
  .tok-id-card {
332
+ display: flex; flex-direction: column; align-items: center; border-radius: 6px;
333
+ overflow: hidden; border: 1px solid; cursor: default;
334
+ transition: transform .15s, box-shadow .15s; min-width: 46px;
 
 
 
 
 
 
 
 
 
 
335
  }
336
+ .tok-id-card:hover { transform: translateY(-2px); box-shadow: 0 4px 16px rgba(0,0,0,.4); }
337
  .tok-id-top {
338
+ padding: 2px 5px; font-family: 'JetBrains Mono', monospace; font-size: 10px;
339
+ font-weight: 500; width: 100%; text-align: center; border-bottom: 1px solid rgba(255,255,255,.08);
 
 
 
 
 
340
  }
341
+ [data-theme="light"] .tok-id-top { border-bottom-color: rgba(0,0,0,.06); }
342
  .tok-id-bottom {
343
+ padding: 1px 5px 2px; font-family: 'JetBrains Mono', monospace; font-size: 8px;
344
+ color: rgba(255,255,255,.4); width: 100%; text-align: center;
 
 
 
 
 
 
 
 
 
 
 
345
  }
346
+ [data-theme="light"] .tok-id-bottom { color: rgba(0,0,0,.35); }
347
+ /* LIST VIEW */
348
+ .token-split-view { display: flex; flex-direction: column; gap: 2px; }
349
  .tok-split-row {
350
+ display: flex; align-items: stretch; border-radius: 5px; overflow: hidden;
351
+ border: 1px solid; font-family: 'JetBrains Mono', monospace; font-size: 11px;
 
 
 
 
 
352
  }
353
  .tok-split-idx {
354
+ width: 34px; text-align: center; padding: 4px 3px; font-size: 9px;
355
+ color: rgba(255,255,255,.3); border-right: 1px solid rgba(255,255,255,.06);
356
+ display: flex; align-items: center; justify-content: center;
 
 
 
 
 
 
 
 
 
 
 
357
  }
358
+ [data-theme="light"] .tok-split-idx { color: rgba(0,0,0,.25); border-right-color: rgba(0,0,0,.06); }
359
+ .tok-split-text { flex: 1; padding: 4px 6px; font-size: 12px; }
360
  .tok-split-id {
361
+ padding: 4px 6px; font-size: 10px; color: rgba(255,255,255,.45);
362
+ border-left: 1px solid rgba(255,255,255,.06); display: flex; align-items: center;
 
 
 
 
363
  }
364
+ [data-theme="light"] .tok-split-id { color: rgba(0,0,0,.35); border-left-color: rgba(0,0,0,.06); }
365
  /* ─── Loading Overlay ────────────────────────────────── */
366
  #loading-overlay {
367
+ position: fixed; inset: 0; background: rgba(6,11,20,.92);
368
+ backdrop-filter: blur(8px); z-index: 1000;
369
+ display: flex; flex-direction: column; align-items: center; justify-content: center;
370
+ gap: 20px; transition: opacity .4s;
 
 
 
 
 
 
 
371
  }
372
+ [data-theme="light"] #loading-overlay { background: rgba(238,241,248,.92); }
373
  #loading-overlay.hidden { opacity: 0; pointer-events: none; }
374
+ .loading-spinner { width: 48px; height: 48px; position: relative; }
375
+ .loading-spinner::before, .loading-spinner::after {
376
+ content: ''; position: absolute; border-radius: 50%; border: 2px solid transparent;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
377
  }
378
+ .loading-spinner::before { inset: 0; border-top-color: var(--accent); animation: spin 1s linear infinite; }
379
+ .loading-spinner::after { inset: 7px; border-top-color: var(--accent2); animation: spin .7s linear infinite reverse; }
380
  @keyframes spin { to { transform: rotate(360deg); } }
381
+ .loading-text { font-family: 'Bricolage Grotesque', sans-serif; font-size: 18px; font-weight: 600; color: var(--text); }
 
 
 
 
 
 
382
  .loading-sub {
383
+ font-family: 'JetBrains Mono', monospace; font-size: 11px; color: var(--text2);
384
+ max-width: 340px; text-align: center;
 
 
 
 
 
 
 
 
 
 
385
  }
386
+ .loading-bar-wrap { width: 260px; height: 3px; background: var(--bg3); border-radius: 2px; overflow: hidden; }
387
  .loading-bar {
388
+ height: 100%; width: 0%;
 
389
  background: linear-gradient(90deg, var(--accent), var(--accent2));
390
+ border-radius: 2px; transition: width .3s;
 
 
 
 
 
 
391
  }
392
+ .loading-file { font-size: 10px; font-family: 'JetBrains Mono', monospace; color: var(--text3); }
393
  /* ─── Error Toast ────────────────────────────────────── */
394
  #toast {
395
+ position: fixed; bottom: 24px; left: 50%;
 
 
396
  transform: translateX(-50%) translateY(80px);
397
+ background: rgba(245,85,119,.15); border: 1px solid rgba(245,85,119,.4);
398
+ color: var(--red); padding: 8px 18px; border-radius: 8px;
399
+ font-size: 12px; font-family: 'JetBrains Mono', monospace;
400
+ z-index: 500; transition: transform .3s; max-width: 460px; text-align: center;
 
 
 
 
 
 
 
401
  }
402
  #toast.show { transform: translateX(-50%) translateY(0); }
 
403
  /* ─── Footer ─────────────────────────────────────────── */
404
  footer {
405
+ padding: 8px 24px; border-top: 1px solid var(--border);
406
+ display: flex; align-items: center; justify-content: space-between;
407
+ font-size: 10px; color: var(--text3); font-family: 'JetBrains Mono', monospace;
408
+ background: rgba(6,11,20,.8); flex-shrink: 0;
 
 
 
 
 
 
 
 
 
 
409
  }
410
+ [data-theme="light"] footer { background: rgba(238,241,248,.8); }
411
+ footer a { color: var(--text2); text-decoration: none; transition: color .15s; }
412
  footer a:hover { color: var(--accent); }
 
413
  /* ─── Scrollbar ──────────────────────────────────────── */
414
+ ::-webkit-scrollbar { width: 5px; }
415
  ::-webkit-scrollbar-track { background: transparent; }
416
  ::-webkit-scrollbar-thumb { background: var(--border); border-radius: 3px; }
417
  ::-webkit-scrollbar-thumb:hover { background: var(--border2); }
418
+ /* ─── Animations ─────────────────────────────────────── */
419
+ @keyframes fadeIn { from { opacity: 0; transform: translateY(4px); } to { opacity: 1; transform: translateY(0); } }
420
+ .fade-in { animation: fadeIn .2s ease forwards; }
 
 
 
 
 
 
 
 
 
 
 
 
 
421
  /* ─── Responsive ─────────────────────────────────────── */
422
+ @media (max-width: 1100px) {
423
+ main { grid-template-columns: 1fr 1fr; }
424
+ main.single-panel { grid-template-columns: 1fr 2fr; }
425
+ .output-panel:last-child { display: none; }
426
+ .searchbar-group:last-of-type { display: none; }
427
+ }
428
+ @media (max-width: 768px) {
429
+ header { padding: 0 12px; gap: 8px; flex-wrap: wrap; height: auto; padding: 8px 12px; }
430
+ .logo-name { font-size: 14px; }
431
  main { grid-template-columns: 1fr; }
432
+ main.single-panel { grid-template-columns: 1fr; }
433
+ .input-panel { border-right: none; border-bottom: 1px solid var(--border); max-height: 35vh; }
434
+ .output-panel { border-left: none !important; }
435
+ .output-panel:last-child { display: none; }
436
+ .searchbar-group { min-width: 140px; }
 
 
 
 
 
 
 
 
437
  }
438
  </style>
439
  </head>
440
  <body>
 
 
441
  <div class="bg-gradient"></div>
442
  <div class="dot-grid"></div>
 
443
  <div id="app">
 
444
  <!-- Header -->
445
  <header>
446
  <div class="logo">
447
  <div class="logo-hex">T</div>
448
  <span class="logo-name">TokenLens</span>
449
+ <span class="logo-tag">v1.2</span>
450
  </div>
451
+ <div class="header-divider"></div>
452
+ <div class="header-controls">
453
+ <!-- Search bar A -->
454
+ <div class="searchbar-group" id="search-group-0">
455
+ <span class="searchbar-label label-a">A</span>
456
+ <input class="searchbar-input" id="search-input-0" type="text" placeholder="HF model id… e.g. Xenova/gpt2" />
457
+ <button class="searchbar-dropdown-btn" id="dropdown-btn-0" title="Predefined models">
458
+ <svg width="10" height="10" viewBox="0 0 10 10" fill="none" stroke="currentColor" stroke-width="1.5"><path d="M2 3.5L5 6.5L8 3.5"/></svg>
459
+ </button>
460
+ <button class="searchbar-load-btn" id="load-btn-0">Load</button>
461
+ <div class="dropdown-menu" id="dropdown-menu-0"></div>
462
+ </div>
463
+ <!-- Search bar B -->
464
+ <div class="searchbar-group" id="search-group-1">
465
+ <span class="searchbar-label label-b">B</span>
466
+ <input class="searchbar-input" id="search-input-1" type="text" placeholder="HF model id… e.g. Xenova/llama-tokenizer" />
467
+ <button class="searchbar-dropdown-btn" id="dropdown-btn-1" title="Predefined models">
468
+ <svg width="10" height="10" viewBox="0 0 10 10" fill="none" stroke="currentColor" stroke-width="1.5"><path d="M2 3.5L5 6.5L8 3.5"/></svg>
469
+ </button>
470
+ <button class="searchbar-load-btn" id="load-btn-1">Load</button>
471
+ <div class="dropdown-menu" id="dropdown-menu-1"></div>
472
+ </div>
473
+ <!-- Toggle: show/hide panel B -->
474
+ <button class="icon-toggle-btn active" id="panel-toggle" title="Toggle comparison panel">
475
+ <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"><rect x="3" y="3" width="7" height="18" rx="1"/><rect x="14" y="3" width="7" height="18" rx="1"/></svg>
476
+ </button>
477
+ <!-- Toggle: light/dark theme -->
478
+ <button class="icon-toggle-btn" id="theme-toggle" title="Toggle light/dark theme">
479
+ <svg id="theme-icon-sun" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" style="display:none"><circle cx="12" cy="12" r="5"/><line x1="12" y1="1" x2="12" y2="3"/><line x1="12" y1="21" x2="12" y2="23"/><line x1="4.22" y1="4.22" x2="5.64" y2="5.64"/><line x1="18.36" y1="18.36" x2="19.78" y2="19.78"/><line x1="1" y1="12" x2="3" y2="12"/><line x1="21" y1="12" x2="23" y2="12"/><line x1="4.22" y1="19.78" x2="5.64" y2="18.36"/><line x1="18.36" y1="5.64" x2="19.78" y2="4.22"/></svg>
480
+ <svg id="theme-icon-moon" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"><path d="M21 12.79A9 9 0 1 1 11.21 3 7 7 0 0 0 21 12.79z"/></svg>
481
+ </button>
482
  </div>
483
  </header>
 
484
  <!-- Main -->
485
+ <main id="main-grid">
 
486
  <!-- Left: Input -->
487
  <div class="input-panel">
488
  <div class="panel-header">
 
493
  <div class="sample-btns">
494
  <button class="sample-btn" data-sample="poetry">Poetry</button>
495
  <button class="sample-btn" data-sample="code">Code</button>
496
+ <button class="sample-btn" data-sample="multilingual">Multi</button>
497
  <button class="sample-btn" data-sample="numbers">Numbers</button>
498
  <button class="sample-btn" data-sample="clear">Clear</button>
499
  </div>
500
  </div>
501
  <textarea id="input-area"
502
+ placeholder="Type or paste text here to see how tokenizers split it into tokens…
503
+
504
+ Try special characters, code, emojis, or multi-lingual text to compare models."></textarea>
505
  <div class="char-counter"><span id="char-count">0</span> characters</div>
506
  </div>
507
+ <!-- Visualizer A -->
508
+ <div class="output-panel" id="panel-0">
509
+ <div class="output-panel-header">
510
+ <div class="model-indicator" id="model-indicator-0">
511
+ <div class="model-indicator-dot" id="model-dot-0" style="background:#3d5a80"></div>
512
+ <span class="model-indicator-name" id="model-label-0">A: no model</span>
513
+ </div>
514
+ </div>
515
  <div class="stats-row">
516
+ <div class="stat-card" id="sc-tokens-0">
517
  <div class="stat-label">Tokens</div>
518
+ <div class="stat-value" id="stat-tokens-0">—</div>
519
+ <div class="stat-sub" id="stat-model-0">no model loaded</div>
520
  </div>
521
+ <div class="stat-card" id="sc-chars-0">
522
  <div class="stat-label">Characters</div>
523
+ <div class="stat-value" id="stat-chars-0">—</div>
524
  <div class="stat-sub">total input</div>
525
  </div>
526
+ <div class="stat-card" id="sc-words-0">
527
  <div class="stat-label">Words</div>
528
+ <div class="stat-value" id="stat-words-0">—</div>
529
  <div class="stat-sub">approx</div>
530
  </div>
531
+ <div class="stat-card" id="sc-ratio-0">
532
+ <div class="stat-label">Chars/Token</div>
533
+ <div class="stat-value" id="stat-ratio-0">—</div>
534
  <div class="stat-sub">efficiency</div>
535
  </div>
536
  </div>
 
 
537
  <div class="view-toggle">
538
+ <div class="toggle-group" id="toggle-group-0">
539
+ <button class="toggle-btn active" data-view="text" data-panel="0">Text View</button>
540
+ <button class="toggle-btn" data-view="ids" data-panel="0">ID Grid</button>
541
+ <button class="toggle-btn" data-view="list" data-panel="0">Token List</button>
542
  </div>
543
+ </div>
544
+ <div class="token-display" id="token-display-0">
545
+ <div class="placeholder-msg" id="placeholder-0">
546
+ <div class="placeholder-icon">⬡</div>
547
+ <p>Load a tokenizer using search bar A above<br>then type text to see tokenization</p>
548
  </div>
549
  </div>
550
+ </div>
551
+ <!-- Visualizer B -->
552
+ <div class="output-panel" id="panel-1">
553
+ <div class="output-panel-header">
554
+ <div class="model-indicator" id="model-indicator-1">
555
+ <div class="model-indicator-dot" id="model-dot-1" style="background:#3d5a80"></div>
556
+ <span class="model-indicator-name" id="model-label-1">B: no model</span>
557
+ </div>
558
+ </div>
559
+ <div class="stats-row">
560
+ <div class="stat-card" id="sc-tokens-1">
561
+ <div class="stat-label">Tokens</div>
562
+ <div class="stat-value" id="stat-tokens-1">—</div>
563
+ <div class="stat-sub" id="stat-model-1">no model loaded</div>
564
+ </div>
565
+ <div class="stat-card" id="sc-chars-1">
566
+ <div class="stat-label">Characters</div>
567
+ <div class="stat-value" id="stat-chars-1">—</div>
568
+ <div class="stat-sub">total input</div>
569
+ </div>
570
+ <div class="stat-card" id="sc-words-1">
571
+ <div class="stat-label">Words</div>
572
+ <div class="stat-value" id="stat-words-1">—</div>
573
+ <div class="stat-sub">approx</div>
574
+ </div>
575
+ <div class="stat-card" id="sc-ratio-1">
576
+ <div class="stat-label">Chars/Token</div>
577
+ <div class="stat-value" id="stat-ratio-1">—</div>
578
+ <div class="stat-sub">efficiency</div>
579
+ </div>
580
+ </div>
581
+ <div class="view-toggle">
582
+ <div class="toggle-group" id="toggle-group-1">
583
+ <button class="toggle-btn active" data-view="text" data-panel="1">Text View</button>
584
+ <button class="toggle-btn" data-view="ids" data-panel="1">ID Grid</button>
585
+ <button class="toggle-btn" data-view="list" data-panel="1">Token List</button>
586
+ </div>
587
+ </div>
588
+ <div class="token-display" id="token-display-1">
589
+ <div class="placeholder-msg" id="placeholder-1">
590
  <div class="placeholder-icon">⬡</div>
591
+ <p>Load a tokenizer using search bar B above<br>then type text to see tokenization</p>
592
  </div>
593
  </div>
594
+ </div>
 
595
  </main>
 
596
  <footer>
597
  <span>TokenLens — Powered by <a href="https://github.com/xenova/transformers.js" target="_blank">Transformers.js</a> · Runs entirely in your browser</span>
598
  <span><a href="https://quickgrid.github.io/">Made by · Asif Ahmed</a></span>
599
  </footer>
600
+ </div>
 
 
601
  <!-- Loading Overlay -->
602
  <div id="loading-overlay">
603
  <div class="loading-spinner"></div>
604
  <div class="loading-text" id="loading-title">Loading Tokenizer</div>
605
+ <div class="loading-sub" id="loading-sub">Downloading tokenizer files from Hugging Face Hub…<br>Cached in your browser after first download.</div>
606
+ <div class="loading-bar-wrap"><div class="loading-bar" id="loading-bar"></div></div>
 
 
607
  <div class="loading-file" id="loading-file"></div>
608
  </div>
 
609
  <!-- Toast -->
610
  <div id="toast"></div>
611
 
 
 
 
 
 
 
 
 
 
612
  <script type="module">
 
613
  import { AutoTokenizer, env }
614
  from 'https://cdn.jsdelivr.net/npm/@xenova/transformers@2.17.2';
615
 
 
616
  env.allowLocalModels = false;
 
617
  env.useBrowserCache = true;
618
 
619
  // ── Model Registry ─────────────────────────────────────────
620
  const MODELS = [
621
+ { id:'Xenova/gpt2', name:'GPT-2', org:'OpenAI', color:'#10a37f', vocab:'50k', type:'BPE', desc:'Classic GPT-2 BPE tokenizer' },
622
+ { id:'Xenova/gpt-4', name:'GPT-4', org:'OpenAI', color:'#10a37f', vocab:'100k', type:'tiktoken cl100k', desc:'Used by GPT-3.5 & GPT-4' },
623
+ { id:'Xenova/llama-tokenizer', name:'LLaMA 2', org:'Meta', color:'#0466de', vocab:'32k', type:'SP-BPE', desc:'SentencePiece BPE — LLaMA / LLaMA-2' },
624
+ { id:'Xenova/mistral-tokenizer-v1', name:'Mistral', org:'Mistral AI', color:'#ff7722', vocab:'32k', type:'SP-BPE', desc:'Mistral 7B v0.1 tokenizer' },
625
+ { id:'Xenova/bert-base-uncased', name:'BERT', org:'Google', color:'#4285f4', vocab:'30k', type:'WordPiece', desc:'BERT-base uncased WordPiece' },
626
+ { id:'Xenova/t5-base', name:'T5', org:'Google', color:'#34a853', vocab:'32k', type:'Unigram', desc:'T5 SentencePiece Unigram' },
627
+ { id:'Xenova/claude-tokenizer', name:'Claude', org:'Anthropic', color:'#cc785c', vocab:'~100k', type:'BPE', desc:"Anthropic Claude's tokenizer" },
628
+ { id:'Xenova/roberta-base', name:'RoBERTa', org:'Meta', color:'#1a73e8', vocab:'50k', type:'BPE', desc:'RoBERTa byte-level BPE' },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
629
  ];
630
 
631
+ // ── Token Color Palettes ───────────────────────────────────
632
+ const PALETTE_DARK = [
633
+ { text:'#ff8080', bg:'rgba(255,128,128,.18)', border:'rgba(255,128,128,.35)' },
634
+ { text:'#ffb84d', bg:'rgba(255,184, 77,.18)', border:'rgba(255,184, 77,.35)' },
635
+ { text:'#ffe066', bg:'rgba(255,224,102,.18)', border:'rgba(255,224,102,.35)' },
636
+ { text:'#7aed91', bg:'rgba(122,237,145,.18)', border:'rgba(122,237,145,.35)' },
637
+ { text:'#4ddfc0', bg:'rgba( 77,223,192,.18)', border:'rgba( 77,223,192,.35)' },
638
+ { text:'#56c8f5', bg:'rgba( 86,200,245,.18)', border:'rgba( 86,200,245,.35)' },
639
+ { text:'#748ef8', bg:'rgba(116,142,248,.18)', border:'rgba(116,142,248,.35)' },
640
+ { text:'#c484f8', bg:'rgba(196,132,248,.18)', border:'rgba(196,132,248,.35)' },
641
+ { text:'#f57cd4', bg:'rgba(245,124,212,.18)', border:'rgba(245,124,212,.35)' },
642
+ { text:'#fa8072', bg:'rgba(250,128,114,.18)', border:'rgba(250,128,114,.35)' },
643
+ { text:'#8be08b', bg:'rgba(139,224,139,.18)', border:'rgba(139,224,139,.35)' },
644
+ { text:'#f0c040', bg:'rgba(240,192, 64,.18)', border:'rgba(240,192, 64,.35)' },
645
+ { text:'#60d4e0', bg:'rgba( 96,212,224,.18)', border:'rgba( 96,212,224,.35)' },
646
+ { text:'#e89060', bg:'rgba(232,144, 96,.18)', border:'rgba(232,144, 96,.35)' },
647
  ];
648
+ const PALETTE_LIGHT = [
649
+ { text:'#cc3333', bg:'rgba(204, 51, 51,.12)', border:'rgba(204, 51, 51,.25)' },
650
+ { text:'#b87218', bg:'rgba(184,114, 24,.12)', border:'rgba(184,114, 24,.25)' },
651
+ { text:'#a08618', bg:'rgba(160,134, 24,.12)', border:'rgba(160,134, 24,.25)' },
652
+ { text:'#228838', bg:'rgba( 34,136, 56,.12)', border:'rgba( 34,136, 56,.25)' },
653
+ { text:'#1a8870', bg:'rgba( 26,136,112,.12)', border:'rgba( 26,136,112,.25)' },
654
+ { text:'#1890b8', bg:'rgba( 24,144,184,.12)', border:'rgba( 24,144,184,.25)' },
655
+ { text:'#3850b8', bg:'rgba( 56, 80,184,.12)', border:'rgba( 56, 80,184,.25)' },
656
+ { text:'#7830a8', bg:'rgba(120, 48,168,.12)', border:'rgba(120, 48,168,.25)' },
657
+ { text:'#b03088', bg:'rgba(176, 48,136,.12)', border:'rgba(176, 48,136,.25)' },
658
+ { text:'#b83828', bg:'rgba(184, 56, 40,.12)', border:'rgba(184, 56, 40,.25)' },
659
+ { text:'#2a882a', bg:'rgba( 42,136, 42,.12)', border:'rgba( 42,136, 42,.25)' },
660
+ { text:'#9a7018', bg:'rgba(154,112, 24,.12)', border:'rgba(154,112, 24,.25)' },
661
+ { text:'#1a8898', bg:'rgba( 26,136,152,.12)', border:'rgba( 26,136,152,.25)' },
662
+ { text:'#a05020', bg:'rgba(160, 80, 32,.12)', border:'rgba(160, 80, 32,.25)' },
663
+ ];
664
+ function getPalette() {
665
+ return document.documentElement.dataset.theme === 'light' ? PALETTE_LIGHT : PALETTE_DARK;
666
+ }
667
 
668
  // ── Sample texts ───────────────────────────────────────────
669
  const SAMPLES = {
670
+ poetry: `Two roads diverged in a yellow wood,\nAnd sorry I could not travel both\nAnd be one traveler, long I stood\nAnd looked down one as far as I could\nTo where it bent in the undergrowth;\n— Robert Frost, "The Road Not Taken"`,
671
+ code: `async function fetchData(url, retries = 3) {\n for (let i = 0; i < retries; i++) {\n try {\n const res = await fetch(url);\n if (!res.ok) throw new Error(\`HTTP \${res.status}\`);\n return await res.json();\n } catch (e) {\n if (i === retries - 1) throw e;\n await new Promise(r => setTimeout(r, 1000 * 2 ** i));\n }\n }\n}`,
672
+ multilingual: `English: The quick brown fox jumps over the lazy dog.\n日本語: 吾輩は猫である。名前はまだない。\n中文: 春眠不觉晓,处处闻啼鸟。\nالعربية: اللغة العربية جميلة ومعبرة.\nΕλληνικά: Η γνώση είναι δύναμη.\nEmoji: 🌍 🦊 ⚡ 🎯 🧬 🤖 🦋`,
673
+ numbers: 3.14159265358979323846\ne 2.71828182845904523536\nφ 1.61803398874989484820\n1,000,000 × $42.99 = $42,990,000.00\n2024-01-15T08:30:00.000Z\nIPv4: 192.168.1.1 | IPv6: ::1`,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
674
  clear: ''
675
  };
676
 
677
  // ── State ──────────────────────────────────────────────────
678
+ const panels = [
679
+ { tokenizer: null, modelId: null, view: 'text' },
680
+ { tokenizer: null, modelId: null, view: 'text' },
681
+ ];
682
+ let tokenizerCache = {};
683
+ let panel1Visible = true;
684
+ let debounceTimer = null;
685
 
686
  // ── DOM References ─────────────────────────────────────────
687
  const $overlay = document.getElementById('loading-overlay');
 
689
  const $loadSub = document.getElementById('loading-sub');
690
  const $loadBar = document.getElementById('loading-bar');
691
  const $loadFile = document.getElementById('loading-file');
 
692
  const $input = document.getElementById('input-area');
693
  const $charCount = document.getElementById('char-count');
 
 
 
 
 
 
 
 
 
694
  const $toast = document.getElementById('toast');
695
+ const $mainGrid = document.getElementById('main-grid');
696
+ const $panelToggle = document.getElementById('panel-toggle');
697
+ const $themeToggle = document.getElementById('theme-toggle');
698
 
699
  // ── Utilities ──────────────────────────────────────────────
 
700
  function showOverlay(title, sub) {
701
  $loadTitle.textContent = title;
702
  $loadSub.textContent = sub;
 
704
  $loadFile.textContent = '';
705
  $overlay.classList.remove('hidden');
706
  }
707
+ function hideOverlay() { $overlay.classList.add('hidden'); }
 
 
 
708
 
709
  function showToast(msg, duration = 5000) {
710
  $toast.textContent = msg;
 
712
  setTimeout(() => $toast.classList.remove('show'), duration);
713
  }
714
 
715
+ function setStats(idx, tokens, text) {
716
  const chars = text.length;
717
  const words = text.trim() ? text.trim().split(/\s+/).length : 0;
718
  const ratio = tokens > 0 && chars > 0 ? (chars / tokens).toFixed(2) : '—';
719
+ document.getElementById(`stat-tokens-${idx}`).textContent = tokens > 0 ? tokens.toLocaleString() : '—';
720
+ document.getElementById(`stat-chars-${idx}`).textContent = chars > 0 ? chars.toLocaleString() : '—';
721
+ document.getElementById(`stat-words-${idx}`).textContent = words > 0 ? words.toLocaleString() : '—';
722
+ document.getElementById(`stat-ratio-${idx}`).textContent = ratio;
723
+ ['tokens','chars','words','ratio'].forEach(k => {
724
+ const el = document.getElementById(`sc-${k}-${idx}`);
725
+ el.classList.remove('highlight'); void el.offsetWidth; el.classList.add('highlight');
 
 
 
 
 
726
  });
727
  }
728
 
729
+ function updateModelIndicator(idx, modelId) {
730
+ const preset = MODELS.find(m => m.id === modelId);
731
+ const color = preset ? preset.color : '#7899c0';
732
+ const name = modelId ? modelId.split('/').pop() : 'no model';
733
+ const label = idx === 0 ? 'A' : 'B';
734
+ document.getElementById(`model-dot-${idx}`).style.background = color;
735
+ document.getElementById(`model-dot-${idx}`).style.boxShadow = `0 0 6px ${color}`;
736
+ document.getElementById(`model-label-${idx}`).textContent = `${label}: ${name}`;
737
+ document.getElementById(`stat-model-${idx}`).textContent = preset ? `${preset.org} · ${preset.type} · ${preset.vocab} vocab` : modelId || 'no model loaded';
738
+ }
739
+
740
  // ── Decode raw token string for display ───────────────────
741
  function decodeTokenString(raw) {
742
  if (!raw) return '';
 
751
  return s;
752
  }
753
 
754
+ // ── Tokenize for a specific panel ─────────────────────────
755
+ async function tokenizeForPanel(idx, text) {
756
+ const p = panels[idx];
757
+ const $display = document.getElementById(`token-display-${idx}`);
758
+ const $placeholder = document.getElementById(`placeholder-${idx}`);
759
+
760
+ if (!p.tokenizer || !text.trim()) {
761
+ const prevView = $display.querySelector('.token-view-container');
762
+ if (prevView) prevView.remove();
763
+ if ($placeholder) $placeholder.style.display = 'flex';
764
+ setStats(idx, 0, text);
765
  return;
766
  }
 
767
  try {
768
+ if ($placeholder) $placeholder.style.display = 'none';
769
+ const encoded = await p.tokenizer(text, { add_special_tokens: false });
 
 
 
 
 
770
  const ids = Array.from(encoded.input_ids.data);
 
771
  let rawTokens;
772
+ try { rawTokens = p.tokenizer.model.convert_ids_to_tokens(ids); }
773
+ catch { rawTokens = await Promise.all(ids.map(id => p.tokenizer.decode([id], { skip_special_tokens: false }))); }
 
 
 
 
 
 
774
  const tokens = ids.map((id, i) => ({
775
+ id, raw: rawTokens[i] || '', display: decodeTokenString(rawTokens[i] || ''),
 
 
776
  }));
777
+ setStats(idx, tokens.length, text);
778
+ renderView(idx, tokens);
 
 
779
  } catch (err) {
780
  console.error('Tokenization error:', err);
781
+ showToast(`Panel ${idx === 0 ? 'A' : 'B'} error: ${err.message}`);
782
  }
783
  }
784
 
785
  // ── Render Views ───────────────────────────────────────────
786
+ function renderView(idx, tokens) {
787
+ const view = panels[idx].view;
788
+ if (view === 'text') renderTextView(idx, tokens);
789
+ else if (view === 'ids') renderIdView(idx, tokens);
790
+ else if (view === 'list') renderListView(idx, tokens);
791
  }
792
 
793
+ function renderTextView(idx, tokens) {
794
+ const PALETTE = getPalette();
795
+ const $display = document.getElementById(`token-display-${idx}`);
796
  const container = document.createElement('div');
797
+ container.className = 'token-text-view token-view-container fade-in';
 
798
  tokens.forEach((tok, i) => {
799
  const c = PALETTE[i % PALETTE.length];
800
  const span = document.createElement('span');
801
  span.className = 'tok';
802
+ span.style.background = c.bg;
803
+ span.style.color = c.text;
804
+ span.style.borderBottom = `2px solid ${c.border}`;
 
805
  const disp = tok.display;
806
+ if (disp === ' ') span.innerHTML = '&nbsp;';
807
+ else if (disp === '\n') span.innerHTML = '↵<br>';
808
+ else if (disp === '\t') span.innerHTML = '→&nbsp;&nbsp;&nbsp;';
809
+ else span.textContent = disp;
 
 
 
 
 
 
810
  const tip = document.createElement('div');
811
  tip.className = 'tok-tooltip';
812
+ const rawEsc = tok.raw.replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;');
813
+ tip.innerHTML = `<span class="tok-tooltip-id">#${tok.id}</span> · <span class="tok-tooltip-text">${rawEsc || '(empty)'}</span>`;
 
 
 
 
 
 
 
814
  span.appendChild(tip);
 
815
  container.appendChild(span);
816
  });
817
+ const prevView = $display.querySelector('.token-view-container');
818
+ if (prevView) prevView.remove();
819
+ document.getElementById(`placeholder-${idx}`).style.display = 'none';
820
  $display.appendChild(container);
821
  }
822
 
823
+ function renderIdView(idx, tokens) {
824
+ const PALETTE = getPalette();
825
+ const $display = document.getElementById(`token-display-${idx}`);
826
  const container = document.createElement('div');
827
+ container.className = 'token-id-view token-view-container fade-in';
 
828
  tokens.forEach((tok, i) => {
829
  const c = PALETTE[i % PALETTE.length];
830
  const card = document.createElement('div');
831
+ card.className = 'tok-id-card';
832
+ card.style.background = c.bg;
833
+ card.style.borderColor = c.border;
834
  card.title = `Raw: ${tok.raw}`;
 
835
  const top = document.createElement('div');
836
+ top.className = 'tok-id-top'; top.style.color = c.text; top.textContent = tok.id;
 
 
 
837
  const bot = document.createElement('div');
838
  bot.className = 'tok-id-bottom';
839
+ bot.textContent = tok.display.slice(0, 8).replace(/\n/g,'↵').replace(/\t/g,'→') || '…';
840
+ card.appendChild(top); card.appendChild(bot);
 
 
 
841
  container.appendChild(card);
842
  });
843
+ const prevView = $display.querySelector('.token-view-container');
844
+ if (prevView) prevView.remove();
845
+ document.getElementById(`placeholder-${idx}`).style.display = 'none';
846
  $display.appendChild(container);
847
  }
848
 
849
+ function renderListView(idx, tokens) {
850
+ const PALETTE = getPalette();
851
+ const $display = document.getElementById(`token-display-${idx}`);
852
  const container = document.createElement('div');
853
+ container.className = 'token-split-view token-view-container fade-in';
 
854
  tokens.forEach((tok, i) => {
855
  const c = PALETTE[i % PALETTE.length];
856
  const row = document.createElement('div');
857
  row.className = 'tok-split-row';
858
  row.style.background = c.bg;
859
  row.style.borderColor = c.border;
860
+ const idxEl = document.createElement('div');
861
+ idxEl.className = 'tok-split-idx'; idxEl.textContent = i;
862
+ const textEl = document.createElement('div');
863
+ textEl.className = 'tok-split-text'; textEl.style.color = c.text;
864
+ textEl.textContent = tok.display.replace(/\n/g,'↵').replace(/\t/g,'→') || '(empty)';
865
+ const idEl = document.createElement('div');
866
+ idEl.className = 'tok-split-id'; idEl.textContent = tok.id;
867
+ row.appendChild(idxEl); row.appendChild(textEl); row.appendChild(idEl);
 
 
 
 
 
 
 
 
 
 
868
  container.appendChild(row);
869
  });
870
+ const prevView = $display.querySelector('.token-view-container');
871
+ if (prevView) prevView.remove();
872
+ document.getElementById(`placeholder-${idx}`).style.display = 'none';
873
  $display.appendChild(container);
874
  }
875
 
876
  // ── Load Tokenizer ─────────────────────────────────────────
877
+ async function loadModel(idx, modelId) {
 
878
  if (tokenizerCache[modelId]) {
879
+ panels[idx].tokenizer = tokenizerCache[modelId];
880
+ panels[idx].modelId = modelId;
881
+ updateModelIndicator(idx, modelId);
882
  await runTokenize();
883
  return;
884
  }
 
885
  const displayName = modelId.split('/').pop();
886
+ const label = idx === 0 ? 'A' : 'B';
887
  showOverlay(
888
+ `Loading ${label}: ${displayName}`,
889
+ `Fetching tokenizer files from Hugging Face Hub.\nCached in IndexedDB after first download.`
890
  );
 
891
  let lastProgress = 0;
 
892
  try {
893
  const tokenizer = await AutoTokenizer.from_pretrained(modelId, {
894
  progress_callback: (info) => {
895
  if (info.status === 'downloading') {
896
+ const pct = info.total ? Math.round((info.loaded / info.total) * 100) : lastProgress;
 
 
897
  $loadBar.style.width = pct + '%';
898
  $loadFile.textContent = info.file || '';
899
  lastProgress = pct;
 
902
  }
903
  }
904
  });
 
905
  tokenizerCache[modelId] = tokenizer;
906
+ panels[idx].tokenizer = tokenizer;
907
+ panels[idx].modelId = modelId;
908
+ updateModelIndicator(idx, modelId);
 
909
  hideOverlay();
910
  await runTokenize();
 
911
  } catch (err) {
912
  hideOverlay();
913
  console.error('Failed to load tokenizer:', err);
914
+ showToast(`Failed to load "${modelId}": ${err.message}`, 8000);
915
  }
916
  }
917
 
918
+ // ── Build Dropdown Menus ───────────────────────────────────
919
+ function buildDropdowns() {
920
+ [0, 1].forEach(idx => {
921
+ const $menu = document.getElementById(`dropdown-menu-${idx}`);
922
+ $menu.innerHTML = '';
923
+ MODELS.forEach(m => {
924
+ const item = document.createElement('div');
925
+ item.className = 'dropdown-item';
926
+ item.innerHTML = `
927
+ <span class="dropdown-item-dot" style="background:${m.color}"></span>
928
+ <span class="dropdown-item-name">${m.name}</span>
929
+ <span class="dropdown-item-detail">${m.org} · ${m.type}</span>
930
+ `;
931
+ item.addEventListener('click', () => {
932
+ const $input = document.getElementById(`search-input-${idx}`);
933
+ $input.value = m.id;
934
+ $menu.classList.remove('open');
935
+ loadModel(idx, m.id);
936
+ });
937
+ $menu.appendChild(item);
938
+ });
 
 
 
 
 
 
 
939
  });
940
  }
941
 
942
+ // ── Dropdown toggle ────────────────────────────────────────
943
+ [0, 1].forEach(idx => {
944
+ const $btn = document.getElementById(`dropdown-btn-${idx}`);
945
+ const $menu = document.getElementById(`dropdown-menu-${idx}`);
946
+ $btn.addEventListener('click', (e) => {
947
+ e.stopPropagation();
948
+ const otherIdx = 1 - idx;
949
+ document.getElementById(`dropdown-menu-${otherIdx}`).classList.remove('open');
950
+ $menu.classList.toggle('open');
951
  });
952
+ });
 
953
 
954
+ document.addEventListener('click', () => {
955
+ document.querySelectorAll('.dropdown-menu').forEach(m => m.classList.remove('open'));
956
+ });
957
+ document.querySelectorAll('.dropdown-menu').forEach(m => {
958
+ m.addEventListener('click', e => e.stopPropagation());
959
+ });
960
 
961
+ // ── Load buttons ───────────────────────────────────────────
962
+ [0, 1].forEach(idx => {
963
+ const $btn = document.getElementById(`load-btn-${idx}`);
964
+ const $input = document.getElementById(`search-input-${idx}`);
965
+ function doLoad() {
966
+ const id = $input.value.trim();
967
+ if (!id) { showToast('Please enter a model ID'); return; }
968
+ loadModel(idx, id);
969
+ }
970
+ $btn.addEventListener('click', doLoad);
971
+ $input.addEventListener('keydown', e => { if (e.key === 'Enter') doLoad(); });
972
+ });
973
+
974
+ // ── View Toggles ───────────────────────────────────────────
975
  document.querySelectorAll('.toggle-btn').forEach(btn => {
976
  btn.addEventListener('click', () => {
977
+ const panelIdx = parseInt(btn.dataset.panel);
978
+ const group = document.getElementById(`toggle-group-${panelIdx}`);
979
+ group.querySelectorAll('.toggle-btn').forEach(b => b.classList.remove('active'));
980
  btn.classList.add('active');
981
+ panels[panelIdx].view = btn.dataset.view;
982
  runTokenize();
983
  });
984
  });
985
 
986
  // ── Input Handling ─────────────────────────────────────────
 
987
  async function runTokenize() {
988
  const text = $input.value;
989
  $charCount.textContent = text.length;
990
+ await Promise.all([
991
+ tokenizeForPanel(0, text),
992
+ panel1Visible ? tokenizeForPanel(1, text) : Promise.resolve()
993
+ ]);
994
  }
995
 
996
+ $input.addEventListener('input', () => {
997
  $charCount.textContent = $input.value.length;
998
  clearTimeout(debounceTimer);
999
  debounceTimer = setTimeout(runTokenize, 280);
1000
  });
1001
 
1002
  // ── Sample Buttons ─────────────────────────────────────────
 
1003
  document.querySelectorAll('.sample-btn').forEach(btn => {
1004
  btn.addEventListener('click', () => {
1005
+ $input.value = SAMPLES[btn.dataset.sample] ?? '';
 
1006
  $input.focus();
1007
  runTokenize();
1008
  });
1009
  });
1010
 
1011
+ // ── Panel Toggle ───────────────────────────────────────────
1012
+ $panelToggle.addEventListener('click', () => {
1013
+ panel1Visible = !panel1Visible;
1014
+ $panelToggle.classList.toggle('active', panel1Visible);
1015
+ const $panel1 = document.getElementById('panel-1');
1016
+ const $searchGroup1 = document.getElementById('search-group-1');
1017
+ if (panel1Visible) {
1018
+ $panel1.style.display = '';
1019
+ $searchGroup1.style.display = '';
1020
+ $mainGrid.classList.remove('single-panel');
1021
+ } else {
1022
+ $panel1.style.display = 'none';
1023
+ $searchGroup1.style.display = 'none';
1024
+ $mainGrid.classList.add('single-panel');
1025
+ }
1026
+ runTokenize();
1027
+ });
1028
 
1029
+ // ── Theme Toggle ───────────────────────────────────────────
1030
+ const $iconSun = document.getElementById('theme-icon-sun');
1031
+ const $iconMoon = document.getElementById('theme-icon-moon');
1032
+
1033
+ function setTheme(theme) {
1034
+ document.documentElement.dataset.theme = theme;
1035
+ if (theme === 'light') {
1036
+ $iconSun.style.display = 'none';
1037
+ $iconMoon.style.display = 'block';
1038
+ } else {
1039
+ $iconSun.style.display = 'block';
1040
+ $iconMoon.style.display = 'none';
1041
+ }
1042
+ runTokenize();
1043
  }
1044
 
1045
+ $themeToggle.addEventListener('click', () => {
1046
+ const current = document.documentElement.dataset.theme;
1047
+ setTheme(current === 'dark' ? 'light' : 'dark');
1048
  });
1049
 
1050
  // ── Init ───────────────────────────────────────────────────
1051
+ buildDropdowns();
1052
+ $overlay.classList.add('hidden');
1053
+ $input.value = '';
1054
 
1055
+ setTheme('dark');
 
 
 
 
 
1056
 
1057
+ document.getElementById('search-input-0').value = MODELS[0].id;
1058
+ loadModel(0, MODELS[0].id);
1059
  </script>
1060
  </body>
1061
+ </html>