Add tokenization details

app.py

@@ -146,12 +146,40 @@ def make_html_block(student_tokenizer, teacher_tokenizer, text, idx):
     )
 
     highlighted = highlight_groups(student_tokenizer, teacher_tokenizer, s_ids, t_ids, s_groups, t_groups)
+
+    # Build tokenized views
+    s_tokens = [student_tokenizer.decode([tid], skip_special_tokens=False, clean_up_tokenization_spaces=False) for tid in s_ids]
+    t_tokens = [teacher_tokenizer.decode([tid], skip_special_tokens=False, clean_up_tokenization_spaces=False) for tid in t_ids]
+
+    s_tokens_html = " | ".join(html.escape(f'"{t}"') for t in s_tokens)
+    t_tokens_html = " | ".join(html.escape(f'"{t}"') for t in t_tokens)
+
+    tokenized_section = f'''
+    <div style="margin-bottom:15px;">
+        <details style="margin-bottom:10px;">
+            <summary style="cursor:pointer; font-weight:bold; user-select:none;">Show tokenization details</summary>
+            <div style="display:grid; grid-template-columns:1fr 1fr; gap:15px; margin-top:10px;">
+                <div style="border:1px solid #ddd; padding:10px; border-radius:5px;">
+                    <strong style="color:#f57c00;">Student Tokens ({len(s_ids)})</strong>
+                    <div style="margin-top:8px; font-size:12px; word-break:break-word;">{s_tokens_html}</div>
+                </div>
+                <div style="border:1px solid #ddd; padding:10px; border-radius:5px;">
+                    <strong style="color:#1976d2;">Teacher Tokens ({len(t_ids)})</strong>
+                    <div style="margin-top:8px; font-size:12px; word-break:break-word;">{t_tokens_html}</div>
+                </div>
+            </div>
+        </details>
+    </div>
+    '''
+
     return (
         f'<div style="border:1px solid #ccc; padding:10px; margin:10px 0; '
         f'border-radius:5px; white-space:pre-wrap; font-family:monospace; font-size:13px;">'
         f"<strong>Text {idx + 1}</strong> "
         f"(student tokens: {len(s_ids)}, teacher tokens: {len(t_ids)})<br><br>"
-        f"{highlighted}</div>"
+        f"{tokenized_section}"
+        f"{highlighted}"
+        f"</div>"
     )