fix(highlight): auto-detect language for code blocks without explicit lang

Code blocks authored without a language rendered as plain text in both the
editor and the published output (Shiki falls back to "text"). Add a shared
lowlight-based detector and use it as a fallback in the editor decorations
and the publish transformer, so language-less blocks get highlighted
identically everywhere - including in already-written articles.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/src/publisher/transformers/highlight-code.ts

@@ -1,5 +1,6 @@
 import type { Element as HastElement, ElementContent, Root as HastRoot } from "hast";
 import { getSharedHighlighter, isSupportedLang, normalizeLang, SHIKI_THEMES } from "../../shared/shiki-config.js";
+import { detectShikiLang } from "../../shared/detect-lang.js";
 import type { Transformer } from "./types.js";
 
 /**
@@ -127,11 +128,15 @@ export const highlightCodeTransformer: Transformer = {
       if (!pre || pre.tagName.toLowerCase() !== "pre") continue;
       if (pre.classList.contains("mermaid")) continue;
 
-      const rawLang = extractLang(codeEl as unknown as Element);
-      const lang = normalizeLang(rawLang);
       const source = codeEl.textContent || "";
       if (!source) continue;
 
+      // Fall back to auto-detection when the block has no explicit language,
+      // so language-less blocks (the common case in authored docs) still get
+      // highlighted. Same logic runs in the editor for an identical result.
+      const rawLang = extractLang(codeEl as unknown as Element);
+      const lang = normalizeLang(rawLang) || detectShikiLang(source);
+
       let hast: HastRoot;
       try {
         hast = highlighter.codeToHast(source, {

backend/src/shared/detect-lang.ts

@@ -0,0 +1,66 @@
+/**
+ * Shared code-language auto-detection.
+ *
+ * Code blocks authored without an explicit language render as plain text in
+ * both the editor (PM decorations) and the publisher (static SSR), because
+ * Shiki falls back to `"text"` when no language is set. This module fills that
+ * gap: given the block's source, it guesses the language with highlight.js
+ * (via lowlight, already a dependency) and maps the guess onto a language
+ * Shiki actually bundles.
+ *
+ * Used by BOTH `code-block-shiki.tsx` (editor) and `highlight-code.ts`
+ * (publisher) so a language-less block is highlighted identically everywhere,
+ * including in articles that were already written.
+ *
+ * Detection results are cached by source text: the editor rebuilds its
+ * decoration set on every doc change, and we don't want to re-run the
+ * (relatively expensive) auto-detector for blocks whose content is unchanged.
+ */
+import { createLowlight, common } from "lowlight";
+import { isSupportedLang, normalizeLang } from "./shiki-config.js";
+
+const lowlight = createLowlight(common);
+
+/** Don't auto-detect on trivially short snippets - too little signal. */
+const MIN_LENGTH = 3;
+/**
+ * highlight.js relevance is roughly proportional to how many language-specific
+ * constructs matched. A small floor avoids tagging arbitrary prose as code
+ * while still catching short-but-clear snippets (e.g. a couple of imports).
+ */
+const MIN_RELEVANCE = 2;
+
+const cache = new Map<string, string>();
+const CACHE_MAX = 200;
+
+/**
+ * Best-effort detection of the Shiki language for a code block that has no
+ * explicit language. Returns a supported Shiki language name, or "" when
+ * detection is inconclusive or lands on a language Shiki doesn't bundle.
+ */
+export function detectShikiLang(code: string | null | undefined): string {
+  const text = code ?? "";
+  if (text.trim().length < MIN_LENGTH) return "";
+
+  const cached = cache.get(text);
+  if (cached !== undefined) return cached;
+
+  let detected = "";
+  try {
+    const result = lowlight.highlightAuto(text);
+    const lang = result.data?.language ?? "";
+    const relevance = result.data?.relevance ?? 0;
+    if (lang && relevance >= MIN_RELEVANCE) {
+      const normalized = normalizeLang(lang);
+      if (isSupportedLang(normalized)) detected = normalized;
+    }
+  } catch {
+    detected = "";
+  }
+
+  // Cheap eviction: clear wholesale once the cache grows past the cap. Code
+  // blocks are few and small, so this rarely triggers.
+  if (cache.size >= CACHE_MAX) cache.clear();
+  cache.set(text, detected);
+  return detected;
+}

backend/tests/highlight-autodetect.test.ts

@@ -0,0 +1,64 @@
+/**
+ * Auto-detection of syntax highlighting for language-less code blocks.
+ *
+ * Code blocks authored without an explicit language must still be highlighted
+ * (in both the editor and the published output). These tests cover the
+ * publisher path end-to-end plus the shared detector in isolation.
+ */
+import { describe, it, expect } from "vitest";
+import { renderArticleHTML, type PublishMeta } from "../src/publisher/html-renderer.js";
+import type { PublishCSS } from "../src/publisher/index.js";
+import { detectShikiLang } from "../src/shared/detect-lang.js";
+
+const EMPTY_CSS: PublishCSS = {
+  variables: "",
+  reset: "",
+  base: "",
+  layout: "",
+  print: "",
+  editorTokens: "",
+  article: "",
+  components: "",
+  publisher: "",
+};
+
+const META: PublishMeta = {
+  title: "Code Article",
+  description: "code highlighting",
+  authors: [{ name: "Alice", affiliationIndices: [1], affiliationNames: ["MIT"] }],
+  affiliations: [{ name: "MIT" }],
+  date: "2025-01-01",
+};
+
+const PY = `import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+def load(name):
+    return AutoModelForCausalLM.from_pretrained(name)`;
+
+describe("detectShikiLang", () => {
+  it("detects python from a typical snippet", () => {
+    expect(detectShikiLang(PY)).toBe("python");
+  });
+
+  it("returns empty for trivially short input", () => {
+    expect(detectShikiLang("x")).toBe("");
+  });
+});
+
+describe("publisher highlights language-less code blocks", () => {
+  it("colors a code block that has no language attribute", async () => {
+    const json = {
+      type: "doc",
+      content: [
+        // Note: no `attrs.language` - mirrors how these blocks are authored.
+        { type: "codeBlock", content: [{ type: "text", text: PY }] },
+      ],
+    };
+    const html = await renderArticleHTML(json as any, META, EMPTY_CSS);
+
+    // Detected language label and per-token color vars must be present.
+    expect(html).toContain('data-lang="python"');
+    expect(html).toContain("--shiki-light:");
+  });
+});

frontend/package-lock.json

@@ -30,6 +30,7 @@
         "@types/katex": "^0.16.8",
         "ai": "^6.0.158",
         "katex": "^0.16.45",
+        "lowlight": "^3.3.0",
         "lucide-react": "^1.8.0",
         "mermaid": "^11.14.0",
         "react": "^18.3.0",
@@ -3834,6 +3835,15 @@
         "url": "https://opencollective.com/unified"
       }
     },
+    "node_modules/highlight.js": {
+      "version": "11.11.1",
+      "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-11.11.1.tgz",
+      "integrity": "sha512-Xwwo44whKBVCYoliBQwaPvtd/2tYFkRQtXDWj1nackaV2JPXx3L0+Jvd8/qCJ2p+ML0/XVkJ2q+Mr+UVdpJK5w==",
+      "license": "BSD-3-Clause",
+      "engines": {
+        "node": ">=12.0.0"
+      }
+    },
     "node_modules/html-url-attributes": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/html-url-attributes/-/html-url-attributes-3.0.1.tgz",
@@ -4095,6 +4105,21 @@
         "loose-envify": "cli.js"
       }
     },
+    "node_modules/lowlight": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/lowlight/-/lowlight-3.3.0.tgz",
+      "integrity": "sha512-0JNhgFoPvP6U6lE/UdVsSq99tn6DhjjpAj5MxG49ewd2mOBVtwWYIT8ClyABhq198aXXODMU6Ox8DrGy/CpTZQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/hast": "^3.0.0",
+        "devlop": "^1.0.0",
+        "highlight.js": "~11.11.0"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/wooorm"
+      }
+    },
     "node_modules/lru-cache": {
       "version": "5.1.1",
       "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",

frontend/package.json

@@ -34,6 +34,7 @@
     "@types/katex": "^0.16.8",
     "ai": "^6.0.158",
     "katex": "^0.16.45",
+    "lowlight": "^3.3.0",
     "lucide-react": "^1.8.0",
     "mermaid": "^11.14.0",
     "react": "^18.3.0",

frontend/src/editor/extensions/code-block-shiki.tsx

@@ -12,6 +12,7 @@ import {
   SHIKI_THEMES,
   type ShikiHighlighter,
 } from "#shared/shiki-config";
+import { detectShikiLang } from "#shared/detect-lang";
 
 /**
  * TipTap code block with Shiki-powered syntax highlighting.
@@ -162,8 +163,10 @@ function buildDecorations({ doc, name, highlighter, defaultLanguage }: Decoratio
 
     if (!highlighter || !text) continue;
 
+    // Auto-detect when no explicit language is set, so language-less blocks
+    // get highlighted in the editor just like in the published output.
     const raw = block.node.attrs.language || defaultLanguage || "";
-    const lang = normalizeLang(raw);
+    const lang = normalizeLang(raw) || detectShikiLang(text);
 
     let hast: HastRoot;
     try {
@@ -274,7 +277,9 @@ export const CodeBlockShiki = CodeBlock.extend<CodeBlockOptions>({
 
       const applyLang = (n: ProsemirrorNode) => {
         const raw = (n.attrs.language as string | null | undefined) || "";
-        const normalized = raw ? normalizeLang(raw) : "";
+        // Mirror the decoration logic: show the auto-detected language label
+        // (and `language-*` class) when the author didn't pick one.
+        const normalized = raw ? normalizeLang(raw) : detectShikiLang(n.textContent || "");
         if (normalized) {
           pre.dataset.lang = normalized;
           code.className = `language-${normalized}`;