index.html

<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Document Classification Leaderboard</title>
<style>
  :root{
    color-scheme:light dark;
    --paper:#eef1f4; --surface:#ffffff; --ink:#17222c; --muted:#5c6b78;
    --hair:#d7dee4; --hair-strong:#c2ccd4;
    --accent:#1d5fa8; --accent-soft:#e5eefa;
    --good:#2f8f6f; --good-soft:#dcefe8; --bad:#c0473b; --warn:#c98a2b;
    --serif:"Iowan Old Style","Palatino Linotype",Palatino,"Book Antiqua",Georgia,serif;
    --sans:system-ui,-apple-system,"Segoe UI",Roboto,Helvetica,Arial,sans-serif;
    --mono:ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
  }
  @media (prefers-color-scheme:dark){
    :root{--paper:#12171c;--surface:#1a2229;--ink:#e6ecf1;--muted:#93a2ae;
      --hair:#2b353e;--hair-strong:#38444e;--accent:#6aa9e6;--accent-soft:#1e2b3a;
      --good:#57b895;--good-soft:#173029;--bad:#e0796d;--warn:#d9a856;}
  }
  *{box-sizing:border-box}
  body{margin:0;background:var(--paper);color:var(--ink);font-family:var(--sans);line-height:1.6;font-size:16px;-webkit-font-smoothing:antialiased}
  .wrap{max-width:1180px;margin:0 auto;padding:clamp(20px,5vw,60px) clamp(18px,6vw,64px)}
  .eyebrow{font-family:var(--mono);font-size:12px;letter-spacing:.18em;text-transform:uppercase;color:var(--accent);font-weight:600;margin:0 0 14px}
  h1{font-family:var(--serif);font-weight:600;font-size:clamp(28px,5vw,44px);line-height:1.08;letter-spacing:-.01em;margin:0 0 14px}
  .lede{font-size:18px;color:var(--muted);max-width:82ch;margin:0 0 8px}
  a{color:var(--accent)}
  code{font-family:var(--mono);font-size:.86em;background:var(--accent-soft);padding:.1em .38em;border-radius:4px}
  .tracknote{background:var(--surface);border:1px solid var(--hair);border-radius:12px;padding:16px 20px;margin:26px 0 6px;font-size:14px}
  .tracknote b{font-family:var(--mono);font-size:12px;letter-spacing:.04em;color:var(--accent)}
  .tracknote ul{margin:8px 0 0;padding-left:18px}.tracknote li{margin:4px 0}
  .tablebox{overflow-x:auto;margin:14px 0}
  table{border-collapse:collapse;width:100%;font-size:14px;min-width:860px}
  th,td{text-align:left;padding:10px 12px;border-bottom:1px solid var(--hair)}
  th{font-family:var(--mono);font-size:11px;letter-spacing:.06em;text-transform:uppercase;color:var(--muted);font-weight:600}
  td.n,th.n{text-align:right;font-variant-numeric:tabular-nums;font-family:var(--mono)}
  tbody tr:last-child td{border-bottom:none}
  tr.ours{background:var(--accent-soft)}
  .rank{font-family:var(--mono);color:var(--muted);width:2ch}
  .name{font-weight:650}
  .chip{display:inline-block;font-family:var(--mono);font-size:10px;font-weight:600;letter-spacing:.03em;padding:2px 7px;border-radius:20px;white-space:nowrap;margin-left:6px}
  .c-com{background:var(--accent-soft);color:var(--accent)}
  .c-open{background:var(--good-soft);color:var(--good)}
  .c-cloud{background:#f0e9dc;color:var(--warn)}
  @media (prefers-color-scheme:dark){.c-cloud{background:#2a2312;color:var(--warn)}}
  .c-gen{background:var(--hair);color:var(--muted)}
  .tabs{display:flex;gap:8px;margin:20px 0 4px}
  .tab{font-family:var(--mono);font-size:12px;font-weight:600;letter-spacing:.03em;padding:7px 14px;border-radius:8px;
    border:1px solid var(--hair-strong);background:var(--surface);color:var(--muted);cursor:pointer}
  .tab.on{background:var(--accent);color:#fff;border-color:var(--accent)}
  .oursonly{color:var(--muted)}
  .bar{position:relative;height:15px;background:var(--hair);border-radius:4px;min-width:110px}
  .bar>span{position:absolute;left:0;top:0;bottom:0;background:var(--accent);border-radius:4px}
  .bar.hl>span{background:var(--good)}
  .barnum{font-family:var(--mono);font-variant-numeric:tabular-nums;font-size:12.5px;margin-left:8px;color:var(--muted)}
  .barcell{display:flex;align-items:center;min-width:200px}
  footer{margin-top:52px;border-top:1px solid var(--hair-strong);padding-top:16px;color:var(--muted);font-size:12.5px;font-family:var(--mono)}
  footer img{vertical-align:middle;margin-right:10px;border-radius:6px}
  .muted{color:var(--muted)}
</style>
</head>
<body>
<div class="wrap">
  <p class="eyebrow">Open-vocab document classification · benchmark</p>
  <h1>Document Classification Leaderboard</h1>
  <p class="lede">Zero-shot document-type classification — labels supplied at inference, not baked into a
    head. Ranked by <strong>doc-type mean macro-F1</strong> across DocLayNet, Forms and Tobacco on the
    held-out
    <a href="https://huggingface.co/datasets/nutrientdocs/document-classification-benchmark">document-classification-benchmark</a>.
    Nutrient's in-house small models vs. cloud frontier VLMs, all scored by the same
    <code>score_classify.py</code> (unweighted macro-F1, argmax over candidate labels).</p>

  <div class="tracknote">
    <b>TRACKS · macro-F1</b>
    <ul>
      <li><strong>DocLayNet / Forms / Tobacco</strong> — document-type tracks (n=377 / 360 / 500); the
        headline <strong>doc-type mean</strong> averages these three and rows are ranked by it.</li>
      <li><strong>OOD</strong> (unseen types) &amp; <strong>OOV</strong> (synonym label wording) — robustness
        tracks, reported for the <em>open-vocab</em> models (Nutrient + the non-VLM generalists). Cloud VLMs
        train on ~all data, so held-out/open-vocab generalization isn't a meaningful axis for them (shown —).</li>
    </ul>
    <p style="margin:10px 0 0"><strong>Reading it:</strong> the in-house models match/lead the cloud on the
      <em>visual</em> tracks (DocLayNet, Forms) at zero API cost, and trail on <em>Tobacco</em> — a
      read-the-header task where a VLM's text reading wins. A dash (—) = track not run for that model.
      <strong>jev (text · OCR)</strong> is a text-only reference (TypeSafe): it classifies from each
      document's OCR/extracted text rather than the image — hence no OOD/OOV coverage.</p>
  </div>

  <div class="tabs">
    <button id="t-nonvlm" class="tab on" onclick="setTab('nonvlm')">vs open-vocab models (non-VLM)</button>
    <button id="t-vlm" class="tab" onclick="setTab('vlm')">vs cloud VLMs</button>
  </div>
  <div id="banner"></div>
  <div class="tablebox"><table><thead id="head"></thead><tbody id="rows"></tbody></table></div>
  <p class="muted" style="font-size:13px">
    <span class="chip c-com">Nutrient · commercial</span>
    <span class="chip c-open">Nutrient · open-weight</span>
    <span class="chip c-cloud">cloud VLM</span>
    &nbsp; Highlighted rows are Nutrient models. Numbers are provisional pending a clean scoring pass.</p>

  <!-- Plain static links so HuggingFace links these model pages (the table links are JS-rendered). -->
  <p id="models-benchmarked" class="muted" style="font-size:12.5px;margin-top:22px">
    <strong>Models benchmarked:</strong>
    <a href="https://huggingface.co/nutrientdocs/document-classification-v2">nutrientdocs/document-classification-v2</a> ·
    <a href="https://huggingface.co/nutrientdocs/document-classification-v1">nutrientdocs/document-classification-v1</a> ·
    <a href="https://huggingface.co/google/siglip2-so400m-patch16-512">google/siglip2-so400m-patch16-512</a> ·
    <a href="https://huggingface.co/google/siglip2-base-patch16-512">google/siglip2-base-patch16-512</a> ·
    <a href="https://huggingface.co/google/siglip-base-patch16-224">google/siglip-base-patch16-224</a> ·
    <a href="https://huggingface.co/openai/clip-vit-large-patch14-336">openai/clip-vit-large-patch14-336</a> ·
    <a href="https://huggingface.co/openai/clip-vit-base-patch32">openai/clip-vit-base-patch32</a> ·
    <a href="https://huggingface.co/laion/CLIP-ViT-H-14-laion2B-s32B-b79K">laion/CLIP-ViT-H-14</a> ·
    <a href="https://huggingface.co/apple/DFN5B-CLIP-ViT-H-14-378">apple/DFN5B-CLIP-ViT-H-14-378</a> ·
    <a href="https://huggingface.co/timm/eva02_large_patch14_clip_336.merged2b_s6b_b61k">timm/eva02_large_patch14_clip_336</a>.
    Cloud VLMs (no public HF page): GPT-4o / 4o-mini / 4.1 / 4.1-mini · Gemini 2.5 Pro / Flash / Flash-Lite ·
    Claude Opus / Sonnet / Haiku ·
    TypeSafe Jev (text-only; classifies from OCR'd document text, not the image).</p>

  <footer>
    <a href="https://nutrient.io/"><img src="https://avatars2.githubusercontent.com/u/1527679?v=3&s=200" height="34" /></a>
    Macro-F1 on nutrientdocs/document-classification-benchmark · same <code>score_classify.py</code> for all ·
    maintained and funded by <a href="https://nutrient.io/">Nutrient</a> ·
    flagship: <a href="https://huggingface.co/nutrientdocs/document-classification-v2">document-classification-v2</a>
  </footer>
</div>

<script>
let DATA=[];
let TAB="nonvlm";   // default: the apples-to-apples open-vocab (non-VLM) fight
function setTab(t){TAB=t;
  document.getElementById('t-nonvlm').className='tab'+(t==='nonvlm'?' on':'');
  document.getElementById('t-vlm').className='tab'+(t==='vlm'?' on':'');
  render();}
const CORE=[["doclaynet","DocLayNet"],["forms","Forms"],["tobacco","Tobacco"]];
const EXTRA=[["ood","OOD*"],["oov","OOV*"]];
const ALL=CORE.concat(EXTRA);
function hf(m){return (m&&m.includes("/"))?"https://huggingface.co/"+m:null;}
function fmt(x){return (x===null||x===undefined)?'<span class="muted">—</span>':x.toFixed(3).replace(/^0/,"");}
function coreMean(t){const v=CORE.map(c=>t[c[0]]).filter(x=>x!==null&&x!==undefined);
  return v.length===CORE.length? v.reduce((a,b)=>a+b,0)/v.length : null;}
function chip(k){return k==="ours-commercial"?'<span class="chip c-com">commercial</span>'
  :k==="ours-open"?'<span class="chip c-open">open</span>'
  :k==="generalist"?'<span class="chip c-gen">generalist</span>':'<span class="chip c-cloud">cloud</span>';}
function render(){
  // full-coverage (all 3 core tracks) ranked by doc-type mean; partial rows after, by DocLayNet
  const D=DATA.filter(r=> TAB==="vlm" ? r.kind!=="generalist" : r.kind!=="cloud");  // ours anchor both tabs
  const full=D.filter(r=>coreMean(r.tracks)!==null).sort((a,b)=>coreMean(b.tracks)-coreMean(a.tracks));
  const part=D.filter(r=>coreMean(r.tracks)===null).sort((a,b)=>(b.tracks.doclaynet||0)-(a.tracks.doclaynet||0));
  const rows=full.concat(part);
  document.getElementById("head").innerHTML='<tr><th class="rank">#</th><th>Model</th>'
    +ALL.map(t=>`<th class="n">${t[1]}</th>`).join("")+'<th class="barcell">Doc-type mean</th><th class="n">Latency</th></tr>';
  const tb=document.getElementById("rows");tb.innerHTML="";
  const max=Math.max(...full.map(r=>coreMean(r.tracks)));
  rows.forEach((r,i)=>{
    const m=coreMean(r.tracks), ours=r.kind&&r.kind.startsWith("ours");
    const tr=document.createElement("tr");if(ours)tr.className="ours";
    const nm=hf(r.model)?`<a class="name" href="${hf(r.model)}" target="_blank" rel="noopener">${r.name}</a>`:`<span class="name">${r.name}</span>`;
    tr.innerHTML=`<td class="rank">${m!==null?i+1:''}</td><td>${nm}${chip(r.chip_kind||r.kind)}</td>`
      +ALL.map(t=>`<td class="n ${EXTRA.some(e=>e[0]===t[0])?'oursonly':''}">${fmt(r.tracks[t[0]])}</td>`).join("")
      +(m!==null?`<td><div class="barcell"><div class="bar ${ours?'hl':''}" style="flex:1"><span style="width:${Math.round(m/max*100)}%"></span></div><span class="barnum">${m.toFixed(3).replace(/^0/,"")}</span></div></td>`
        :`<td class="muted" style="font-size:12px">partial coverage</td>`)
      +`<td class="n">${r.latency||'<span class="muted">—</span>'}</td>`;
    tb.appendChild(tr);
  });
}
function setBanner(){const b=document.getElementById("banner");
  if(DATA.some(d=>d.provisional)){b.className="tracknote";b.style.borderColor="var(--warn)";b.textContent="Provisional numbers — a clean scoring pass on standard hardware is pending.";}}
fetch("leaderboard.json").then(r=>r.ok?r.json():[]).then(d=>{DATA=Array.isArray(d)?d:(d.results||[]);render();}).catch(()=>render());
</script>
</body>
</html>