aggregate.py: unified-512 intro + per-class + Wilcoxon

code/framework/report/aggregate.py

@@ -188,12 +188,20 @@ def _intro_html():
     h.append("<h2>Project overview</h2>")
     h.append("<p>A unified benchmark of <b>8 2D medical-image segmentation methods</b> across "
              "<b>10 public datasets</b> spanning <b>7 imaging modalities</b> (endoscopy, retinal fundus, "
-             "ultrasound, cardiac MRI, dermoscopy, histopathology, abdominal CT). Every method is trained "
-             "and evaluated through one pipeline (bf16 AMP on A100, identical metrics). Reported as "
+             "ultrasound, cardiac MRI, dermoscopy, histopathology, abdominal CT). Every method runs through "
+             "one pipeline with identical metrics. Reported as "
              "<b>mean±SD</b> over <b>3 seeds</b> for fixed-split datasets and over <b>folds</b> for "
              "cross-validation datasets (PanNuke: official 3-fold). Each (dataset,method) cell aggregates "
              "~tens–thousands of test images; the suite totals ≈20k images. Per-method efficiency "
              "(params / FLOPs / throughput) is in <code>efficiency.md</code>.</p>")
+    h.append("<p style='background:#eef7f0;border-left:3px solid #0a6;padding:8px 12px'>"
+             "<b>Resolution-fair evaluation (unified 512).</b> Convolutional methods are trained at "
+             "512×512; the fixed-input transformers (Swin-UNet 224, TransUNet 256) run at their native "
+             "size; nnU-Net / U-Mamba predict natively. <b>For scoring, every method's prediction and the "
+             "ground truth are resized to a common 512×512</b> before computing all 7 metrics — so the "
+             "boundary metrics (HD95 / ASSD), which are measured in pixels, are <b>directly comparable "
+             "across methods</b> regardless of each method's native working resolution. (Earlier 256-px "
+             "tables mixed pixel scales and were not boundary-comparable.)</p>")
 
     h.append("<h2>Datasets</h2>")
     h.append("<table><tr><th>#</th><th>Dataset</th><th>Modality</th><th>Target</th><th>Classes</th>"
@@ -238,7 +246,133 @@ def _intro_html():
     return "\n".join(h)
 
 
-def to_html(rows, title="SegGen baselines"):
+# ---- (3) per-class Dice tables for the multi-class datasets ----
+# Class names follow each dataset's standard foreground labelling (0=background excluded).
+_PERCLASS_NAMES = {
+    "acdc_png": {"1": "RV", "2": "Myocardium", "3": "LV"},
+    "refuge2": {"1": "Optic Disc", "2": "Optic Cup"},
+    "idridd_segmentation": {"1": "MA", "2": "Haemorrhage", "3": "Hard Exudate", "4": "Soft Exudate", "5": "Optic Disc"},
+    "pannuke_semantic": {"1": "Neoplastic", "2": "Inflammatory", "3": "Connective", "4": "Dead", "5": "Epithelial"},
+}
+
+
+def _collect_perclass(runs):
+    acc = defaultdict(lambda: defaultdict(list))   # (dataset,arch)->class->[dice]
+    for d in runs:
+        key = (d.get("dataset"), d.get("arch"))
+        for pi in d.get("per_image", []):
+            for c, m in (pi.get("per_class") or {}).items():
+                v = (m or {}).get("dice")
+                if v is not None and v == v:
+                    acc[key][c].append(v)
+    return {k: {c: float(np.mean(v)) for c, v in cd.items() if v} for k, cd in acc.items()}
+
+
+def _perclass_section(runs):
+    pc = _collect_perclass(runs)
+    h = ["<h1 style='font-size:18px'>Per-class Dice — multi-class datasets</h1>",
+         "<div class='cap'>Mean per-class Dice (%) over all test images and all runs (0=background "
+         "excluded). Reveals which structures/types drive a method's macro-Dice. Class names follow each "
+         "dataset's standard labelling (IDRiD: MA=microaneurysm, exudates hard/soft, optic disc).</div>"]
+    for ds, names in _PERCLASS_NAMES.items():
+        methods = [a for a in _ARCH_ORDER if (ds, a) in pc and pc[(ds, a)]]
+        if not methods:
+            continue
+        classes = sorted(names, key=int)
+        h.append(f"<h2>{ds}</h2>")
+        h.append("<table><tr><th>Method</th>" + "".join(f"<th>{names[c]}</th>" for c in classes)
+                 + "<th>macro</th></tr>")
+        # best per class (column) for bolding
+        colbest = {c: max((pc[(ds, a)].get(c, float('nan')) for a in methods),
+                          default=float('nan')) for c in classes}
+        for a in methods:
+            cd = pc[(ds, a)]
+            cells = []
+            present = []
+            for c in classes:
+                v = cd.get(c)
+                if v is None:
+                    cells.append("<td>—</td>")
+                else:
+                    present.append(v)
+                    txt = f"{v*100:.1f}"
+                    cells.append(f"<td>{'<b>'+txt+'</b>' if v == colbest[c] else txt}</td>")
+            macro = (sum(present) / len(present) * 100) if present else float("nan")
+            h.append(f"<tr><td class='l'>{a}</td>{''.join(cells)}<td>{macro:.1f}</td></tr>")
+        h.append("</table>")
+    return "\n".join(h)
+
+
+# ---- (4) paired Wilcoxon significance on per-image Dice ----
+def _per_image_dice_vec(runs_for_da):
+    """Aligned per-image Dice for one (dataset,arch): mean over seeds within each
+    protocol/fold, concatenated over sorted protocols. Length matches across methods."""
+    by_proto = defaultdict(list)
+    for d in runs_for_da:
+        by_proto[d.get("protocol")].append(d)
+    parts = []
+    for proto in sorted(by_proto):
+        arrs = [np.array([pi.get("dice", np.nan) for pi in d.get("per_image", [])], float)
+                for d in by_proto[proto]]
+        arrs = [a for a in arrs if a.size]
+        if not arrs:
+            continue
+        L = min(a.size for a in arrs)
+        parts.append(np.nanmean(np.stack([a[:L] for a in arrs]), axis=0))
+    return np.concatenate(parts) if parts else np.array([])
+
+
+def _wilcoxon_section(runs):
+    try:
+        from scipy.stats import wilcoxon
+    except Exception:
+        return "<p>(scipy unavailable — significance section skipped)</p>"
+    by_da = defaultdict(list)
+    for d in runs:
+        by_da[(d.get("dataset"), d.get("arch"))].append(d)
+
+    def pval(a, b):
+        L = min(a.size, b.size)
+        if L < 6:
+            return float("nan")
+        x, y = a[:L], b[:L]
+        m = ~(np.isnan(x) | np.isnan(y))
+        if m.sum() < 6 or np.allclose(x[m], y[m]):
+            return 1.0
+        try:
+            return float(wilcoxon(x[m], y[m]).pvalue)
+        except Exception:
+            return 1.0
+
+    h = ["<h1 style='font-size:18px'>Statistical significance — paired Wilcoxon (per-image Dice)</h1>",
+         "<div class='cap'>Per dataset, the best method by mean Dice is marked ★; a paired Wilcoxon "
+         "signed-rank test on per-image Dice (paired by image) compares it against every other method. "
+         "Methods <u>not</u> significantly worse than the best (p≥0.05) are <u>underlined</u> = statistically "
+         "tied for best. The last column is the best-vs-2nd-best p-value. Pairing is exact for the in-framework "
+         "methods (shared dataloader order); nnU-Net/U-Mamba use cached predictions paired by index where "
+         "image counts match.</div>"]
+    h.append("<table><tr><th>Dataset</th><th>Best ★ and tied-for-best (underlined)</th>"
+             "<th>best vs 2nd p</th></tr>")
+    for ds in sorted({k[0] for k in by_da}):
+        vecs = {a: _per_image_dice_vec(by_da[(ds, a)]) for a in _ARCH_ORDER if (ds, a) in by_da}
+        vecs = {a: v for a, v in vecs.items() if v.size}
+        if len(vecs) < 2:
+            continue
+        means = {a: float(np.nanmean(v)) for a, v in vecs.items()}
+        ranked = sorted(means, key=means.get, reverse=True)
+        best = ranked[0]
+        tied = [a for a in ranked[1:] if not (pval(vecs[best], vecs[a]) < 0.05)]
+        p2 = pval(vecs[best], vecs[ranked[1]])
+        marks = f"<b>★ {best} ({means[best]*100:.1f})</b>"
+        if tied:
+            marks += " &nbsp;·&nbsp; tied: " + ", ".join(f"<u>{a}</u>" for a in tied)
+        h.append(f"<tr><td class='l'>{ds}</td><td class='l'>{marks}</td>"
+                 f"<td>{('%.2g' % p2) if p2 == p2 else '—'}</td></tr>")
+    h.append("</table>")
+    return "\n".join(h)
+
+
+def to_html(rows, runs=None, title="SegGen baselines"):
     cell, dslist = {}, []
     for r in rows:
         ds = f"{r['dataset']}/{r['protocol']}"
@@ -282,6 +416,11 @@ def to_html(rows, title="SegGen baselines"):
             tds.append(f"<td>{nseeds.get(ds,'')}</td>")
             h.append("<tr>" + "".join(tds) + "</tr>")
         h.append("</table>")
+    if runs:
+        h.append("<hr style='margin:22px 0;border:none;border-top:2px solid #0a6'>")
+        h.append(_perclass_section(runs))
+        h.append("<hr style='margin:22px 0;border:none;border-top:2px solid #0a6'>")
+        h.append(_wilcoxon_section(runs))
     h.append("</body></html>")
     return "\n".join(h)
 
@@ -301,7 +440,7 @@ def main():
     open(os.path.join(base, "summary.csv"), "w").write(to_csv(rows))
     open(os.path.join(base, "summary.md"), "w").write(to_markdown(rows))
     open(os.path.join(base, "summary.tex"), "w").write(to_latex(rows))
-    open(os.path.join(base, "summary.html"), "w").write(to_html(rows, title=f"SegGen baselines ({args.exp_name})"))
+    open(os.path.join(base, "summary.html"), "w").write(to_html(rows, runs, title=f"SegGen baselines ({args.exp_name})"))
     print(to_markdown(rows))
     print(f"{len(runs)} runs -> {len(rows)} (dataset,arch) cells; written {base}/summary.{{csv,md,tex,html}}")