# Builds a visual PDF report from reports/metrics.json and # reports/run_metadata.json - same numbers as the memo, laid out so # someone can understand the results without reading raw JSON. from __future__ import annotations import json from pathlib import Path import matplotlib import matplotlib.pyplot as plt from matplotlib.backends.backend_pdf import PdfPages matplotlib.rcParams.update({ "font.family": "sans-serif", "font.size": 11, "text.color": "#0b0b0b", "axes.edgecolor": "#0b0b0b", "axes.labelcolor": "#0b0b0b", "xtick.color": "#0b0b0b", "ytick.color": "#0b0b0b", }) BLUE = "#2a78d6" # validated single hue, see dataviz skill palette GRID = "#d9d9d6" # recessive gridline MUTED = "#52514e" # secondary text ROOT = Path(__file__).resolve().parent.parent metrics = json.loads((ROOT / "reports/metrics.json").read_text()) run = json.loads((ROOT / "reports/run_metadata.json").read_text()) CLASS_INSTANCES = { # from the evaluate.py console output for this run "Caption": 149, "Footnote": 47, "Formula": 150, "List-item": 962, "Page-footer": 408, "Page-header": 311, "Picture": 143, "Section-header": 873, "Table": 252, "Text": 3002, "Title": 52, } def new_page(figsize=(8.5, 11)): fig = plt.figure(figsize=figsize, facecolor="white") fig.patch.set_facecolor("white") return fig def bare_axes(fig, rect): ax = fig.add_axes(rect) ax.set_facecolor("white") for spine in ax.spines.values(): spine.set_visible(False) ax.set_xticks([]) ax.set_yticks([]) return ax def page_overview(): fig = new_page() ax = bare_axes(fig, [0, 0, 1, 1]) ax.text(0.07, 0.94, "Document Layout Detection — Evaluation Report", fontsize=20, fontweight="bold", ha="left") ax.text(0.07, 0.905, "RT-DETR-L fine-tuned on DocLayNet, evaluated on the held-out test split", fontsize=12, color=MUTED, ha="left") ax.axhline(0.885, xmin=0.07, xmax=0.93, color="#0b0b0b", linewidth=1) overall = metrics["overall"] stats = [ ("mAP50", f"{overall['mAP50']:.3f}"), ("mAP50-95", f"{overall['mAP50_95']:.3f}"), ("Precision", f"{overall['precision']:.3f}"), ("Recall", f"{overall['recall']:.3f}"), ] x0 = 0.07 box_w = 0.20 for i, (label, value) in enumerate(stats): x = x0 + i * (box_w + 0.013) ax.add_patch(plt.Rectangle((x, 0.74), box_w, 0.12, fill=False, edgecolor="#0b0b0b", linewidth=1)) ax.text(x + box_w / 2, 0.815, value, fontsize=22, fontweight="bold", ha="center", va="center", color=BLUE) ax.text(x + box_w / 2, 0.755, label, fontsize=10.5, ha="center", va="center", color=MUTED) ax.text(0.07, 0.68, "Test set: 499 pages, 6,349 annotated regions, 11 classes (none in COCO)", fontsize=10.5, color=MUTED) # metric bar chart - 4 distinct measures of the same model, so a small # categorical set (not a repeated single-series magnitude chart) cats = ["Precision", "Recall", "mAP50", "mAP50-95"] vals = [overall["precision"], overall["recall"], overall["mAP50"], overall["mAP50_95"]] colors = ["#2a78d6", "#eb6834", "#1baf7a", "#eda100"] # fixed categorical order, slots 1-4 ax2 = fig.add_axes([0.10, 0.48, 0.80, 0.16]) bars = ax2.bar(cats, vals, color=colors, width=0.55) ax2.set_ylim(0, 1.0) ax2.grid(axis="y", color=GRID, linewidth=0.8, zorder=0) ax2.set_axisbelow(True) for spine in ("top", "right", "left"): ax2.spines[spine].set_visible(False) ax2.spines["bottom"].set_color("#0b0b0b") ax2.tick_params(left=False) ax2.set_yticklabels([]) for bar, v in zip(bars, vals): ax2.text(bar.get_x() + bar.get_width() / 2, v + 0.02, f"{v:.3f}", ha="center", fontsize=10, fontweight="bold") # training config, factual, no names/dates hp = run["hyperparameters"] hw = run["hardware"] config_lines = [ f"Epochs: {hp['epochs']} Batch: {hp['batch']} Image size: {hp['imgsz']}px", f"Optimizer: {hp['optimizer']} LR: {hp['lr0']} Seed: {hp['seed']}", f"Hardware: {hw['gpu']} ({hw['gpu_memory_gb']} GB) Wall clock: {run['wall_clock_human']}", f"Query-budget saturation: {metrics['query_saturation']['percent_over_budget']}% of test pages " f"(max {metrics['query_saturation']['max_regions_on_any_page']} regions, budget " f"{metrics['query_saturation']['query_budget']})", f"Train/test source-PDF overlap: {metrics['split_leakage']['percent_affected']}% " f"({metrics['split_leakage']['test_pages_total']} test pages checked)", ] ax.text(0.07, 0.40, "Training configuration", fontsize=13, fontweight="bold") for i, line in enumerate(config_lines): ax.text(0.07, 0.365 - i * 0.032, line, fontsize=10.5, color="#0b0b0b") return fig def page_per_class(): fig = new_page() ax_title = bare_axes(fig, [0, 0, 1, 1]) ax_title.text(0.07, 0.955, "Per-class performance (mAP50)", fontsize=16, fontweight="bold") ax_title.text(0.07, 0.928, "Sorted by score. Instance count shown per class — rarity, not size alone, " "drives the weakest results.", fontsize=10, color=MUTED) per_class = metrics["overall"] # placeholder, real data below items = sorted( [(name, CLASS_INSTANCES[name]) for name in CLASS_INSTANCES], key=lambda kv: PER_CLASS_MAP50[kv[0]], ) names = [n for n, _ in items] values = [PER_CLASS_MAP50[n] for n, _ in items] counts = [c for _, c in items] ax = fig.add_axes([0.28, 0.08, 0.62, 0.80]) y = range(len(names)) ax.barh(y, values, color=BLUE, height=0.6, zorder=3) ax.set_yticks(list(y)) ax.set_yticklabels(names, fontsize=11) ax.set_xlim(0, 1.0) ax.grid(axis="x", color=GRID, linewidth=0.8, zorder=0) ax.set_axisbelow(True) for spine in ("top", "right"): ax.spines[spine].set_visible(False) ax.spines["left"].set_color("#0b0b0b") ax.spines["bottom"].set_color("#0b0b0b") ax.set_xlabel("mAP50", fontsize=10.5, color=MUTED) for yi, (v, c) in enumerate(zip(values, counts)): ax.text(v + 0.015, yi, f"{v:.3f}", va="center", fontsize=9.5, fontweight="bold") ax.text(1.02, yi, f"n={c}", va="center", fontsize=8.5, color=MUTED, transform=ax.get_yaxis_transform()) return fig def page_per_category(): fig = new_page() ax_title = bare_axes(fig, [0, 0, 1, 1]) ax_title.text(0.07, 0.955, "Per-document-category performance (mAP50)", fontsize=16, fontweight="bold") ax_title.text(0.07, 0.928, "Tests generalization across document styles, not just aggregate accuracy.", fontsize=10, color=MUTED) cat = metrics["per_doc_category"] items = sorted(cat.items(), key=lambda kv: kv[1]["mAP50"]) names = [k.replace("_", " ") for k, _ in items] values = [v["mAP50"] for _, v in items] pages = [v["pages"] for _, v in items] ax = fig.add_axes([0.30, 0.55, 0.60, 0.34]) y = range(len(names)) ax.barh(y, values, color=BLUE, height=0.55, zorder=3) ax.set_yticks(list(y)) ax.set_yticklabels(names, fontsize=11) ax.set_xlim(0, 1.0) ax.grid(axis="x", color=GRID, linewidth=0.8, zorder=0) ax.set_axisbelow(True) for spine in ("top", "right"): ax.spines[spine].set_visible(False) ax.spines["left"].set_color("#0b0b0b") ax.spines["bottom"].set_color("#0b0b0b") ax.set_xlabel("mAP50", fontsize=10.5, color=MUTED) for yi, (v, p) in enumerate(zip(values, pages)): ax.text(v + 0.015, yi, f"{v:.3f} ({p} pages)", va="center", fontsize=9.5, fontweight="bold") # failure notes, factual only ax_title.text(0.07, 0.42, "Observed failure patterns", fontsize=13, fontweight="bold") notes = [ "Footnote (47 instances): recall 0.048 — rare class, near-total miss rate.", "Page-footer (408 instances): mAP50 0.843 — same thin shape as Footnote, but", " 9x more training instances and a consistent position. Rarity, not size,", " is the dominant factor.", "Picture: precision 0.416 despite recall 0.622 — composite regions (diagrams", " with embedded text) get split into multiple overlapping boxes instead", " of one region.", "Dense repeated-entry layouts (directories, org charts) produce duplicate,", " overlapping box proposals rather than one box per entry.", ] for i, line in enumerate(notes): ax_title.text(0.07, 0.385 - i * 0.028, line, fontsize=10, color="#0b0b0b") return fig PER_CLASS_MAP50 = { "Caption": 0.622, "Footnote": 0.182, "Formula": 0.788, "List-item": 0.671, "Page-footer": 0.843, "Page-header": 0.658, "Picture": 0.495, "Section-header": 0.762, "Table": 0.761, "Text": 0.832, "Title": 0.224, } if __name__ == "__main__": import sys preview = "--preview" in sys.argv out_path = ROOT / "reports" / "evaluation_report.pdf" pages = [page_overview(), page_per_class(), page_per_category()] with PdfPages(out_path, metadata={ "Title": "", "Author": "", "Subject": "", "Creator": "", "Producer": "", "CreationDate": None, }) as pdf: for i, fig in enumerate(pages, start=1): pdf.savefig(fig) if preview: fig.savefig(ROOT / f"reports/_preview_page{i}.png", dpi=110) plt.close(fig) print(f"wrote {out_path}")