Spaces:
Running on Zero
Running on Zero
File size: 9,542 Bytes
087643a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 | # Builds a visual PDF report from reports/metrics.json and
# reports/run_metadata.json - same numbers as the memo, laid out so
# someone can understand the results without reading raw JSON.
from __future__ import annotations
import json
from pathlib import Path
import matplotlib
import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages
matplotlib.rcParams.update({
"font.family": "sans-serif",
"font.size": 11,
"text.color": "#0b0b0b",
"axes.edgecolor": "#0b0b0b",
"axes.labelcolor": "#0b0b0b",
"xtick.color": "#0b0b0b",
"ytick.color": "#0b0b0b",
})
BLUE = "#2a78d6" # validated single hue, see dataviz skill palette
GRID = "#d9d9d6" # recessive gridline
MUTED = "#52514e" # secondary text
ROOT = Path(__file__).resolve().parent.parent
metrics = json.loads((ROOT / "reports/metrics.json").read_text())
run = json.loads((ROOT / "reports/run_metadata.json").read_text())
CLASS_INSTANCES = { # from the evaluate.py console output for this run
"Caption": 149, "Footnote": 47, "Formula": 150, "List-item": 962,
"Page-footer": 408, "Page-header": 311, "Picture": 143,
"Section-header": 873, "Table": 252, "Text": 3002, "Title": 52,
}
def new_page(figsize=(8.5, 11)):
fig = plt.figure(figsize=figsize, facecolor="white")
fig.patch.set_facecolor("white")
return fig
def bare_axes(fig, rect):
ax = fig.add_axes(rect)
ax.set_facecolor("white")
for spine in ax.spines.values():
spine.set_visible(False)
ax.set_xticks([])
ax.set_yticks([])
return ax
def page_overview():
fig = new_page()
ax = bare_axes(fig, [0, 0, 1, 1])
ax.text(0.07, 0.94, "Document Layout Detection — Evaluation Report",
fontsize=20, fontweight="bold", ha="left")
ax.text(0.07, 0.905, "RT-DETR-L fine-tuned on DocLayNet, evaluated on the held-out test split",
fontsize=12, color=MUTED, ha="left")
ax.axhline(0.885, xmin=0.07, xmax=0.93, color="#0b0b0b", linewidth=1)
overall = metrics["overall"]
stats = [
("mAP50", f"{overall['mAP50']:.3f}"),
("mAP50-95", f"{overall['mAP50_95']:.3f}"),
("Precision", f"{overall['precision']:.3f}"),
("Recall", f"{overall['recall']:.3f}"),
]
x0 = 0.07
box_w = 0.20
for i, (label, value) in enumerate(stats):
x = x0 + i * (box_w + 0.013)
ax.add_patch(plt.Rectangle((x, 0.74), box_w, 0.12, fill=False,
edgecolor="#0b0b0b", linewidth=1))
ax.text(x + box_w / 2, 0.815, value, fontsize=22, fontweight="bold",
ha="center", va="center", color=BLUE)
ax.text(x + box_w / 2, 0.755, label, fontsize=10.5, ha="center",
va="center", color=MUTED)
ax.text(0.07, 0.68, "Test set: 499 pages, 6,349 annotated regions, 11 classes (none in COCO)",
fontsize=10.5, color=MUTED)
# metric bar chart - 4 distinct measures of the same model, so a small
# categorical set (not a repeated single-series magnitude chart)
cats = ["Precision", "Recall", "mAP50", "mAP50-95"]
vals = [overall["precision"], overall["recall"], overall["mAP50"], overall["mAP50_95"]]
colors = ["#2a78d6", "#eb6834", "#1baf7a", "#eda100"] # fixed categorical order, slots 1-4
ax2 = fig.add_axes([0.10, 0.48, 0.80, 0.16])
bars = ax2.bar(cats, vals, color=colors, width=0.55)
ax2.set_ylim(0, 1.0)
ax2.grid(axis="y", color=GRID, linewidth=0.8, zorder=0)
ax2.set_axisbelow(True)
for spine in ("top", "right", "left"):
ax2.spines[spine].set_visible(False)
ax2.spines["bottom"].set_color("#0b0b0b")
ax2.tick_params(left=False)
ax2.set_yticklabels([])
for bar, v in zip(bars, vals):
ax2.text(bar.get_x() + bar.get_width() / 2, v + 0.02, f"{v:.3f}",
ha="center", fontsize=10, fontweight="bold")
# training config, factual, no names/dates
hp = run["hyperparameters"]
hw = run["hardware"]
config_lines = [
f"Epochs: {hp['epochs']} Batch: {hp['batch']} Image size: {hp['imgsz']}px",
f"Optimizer: {hp['optimizer']} LR: {hp['lr0']} Seed: {hp['seed']}",
f"Hardware: {hw['gpu']} ({hw['gpu_memory_gb']} GB) Wall clock: {run['wall_clock_human']}",
f"Query-budget saturation: {metrics['query_saturation']['percent_over_budget']}% of test pages "
f"(max {metrics['query_saturation']['max_regions_on_any_page']} regions, budget "
f"{metrics['query_saturation']['query_budget']})",
f"Train/test source-PDF overlap: {metrics['split_leakage']['percent_affected']}% "
f"({metrics['split_leakage']['test_pages_total']} test pages checked)",
]
ax.text(0.07, 0.40, "Training configuration", fontsize=13, fontweight="bold")
for i, line in enumerate(config_lines):
ax.text(0.07, 0.365 - i * 0.032, line, fontsize=10.5, color="#0b0b0b")
return fig
def page_per_class():
fig = new_page()
ax_title = bare_axes(fig, [0, 0, 1, 1])
ax_title.text(0.07, 0.955, "Per-class performance (mAP50)", fontsize=16, fontweight="bold")
ax_title.text(0.07, 0.928, "Sorted by score. Instance count shown per class — rarity, not size alone, "
"drives the weakest results.", fontsize=10, color=MUTED)
per_class = metrics["overall"] # placeholder, real data below
items = sorted(
[(name, CLASS_INSTANCES[name]) for name in CLASS_INSTANCES],
key=lambda kv: PER_CLASS_MAP50[kv[0]],
)
names = [n for n, _ in items]
values = [PER_CLASS_MAP50[n] for n, _ in items]
counts = [c for _, c in items]
ax = fig.add_axes([0.28, 0.08, 0.62, 0.80])
y = range(len(names))
ax.barh(y, values, color=BLUE, height=0.6, zorder=3)
ax.set_yticks(list(y))
ax.set_yticklabels(names, fontsize=11)
ax.set_xlim(0, 1.0)
ax.grid(axis="x", color=GRID, linewidth=0.8, zorder=0)
ax.set_axisbelow(True)
for spine in ("top", "right"):
ax.spines[spine].set_visible(False)
ax.spines["left"].set_color("#0b0b0b")
ax.spines["bottom"].set_color("#0b0b0b")
ax.set_xlabel("mAP50", fontsize=10.5, color=MUTED)
for yi, (v, c) in enumerate(zip(values, counts)):
ax.text(v + 0.015, yi, f"{v:.3f}", va="center", fontsize=9.5, fontweight="bold")
ax.text(1.02, yi, f"n={c}", va="center", fontsize=8.5, color=MUTED, transform=ax.get_yaxis_transform())
return fig
def page_per_category():
fig = new_page()
ax_title = bare_axes(fig, [0, 0, 1, 1])
ax_title.text(0.07, 0.955, "Per-document-category performance (mAP50)", fontsize=16, fontweight="bold")
ax_title.text(0.07, 0.928, "Tests generalization across document styles, not just aggregate accuracy.",
fontsize=10, color=MUTED)
cat = metrics["per_doc_category"]
items = sorted(cat.items(), key=lambda kv: kv[1]["mAP50"])
names = [k.replace("_", " ") for k, _ in items]
values = [v["mAP50"] for _, v in items]
pages = [v["pages"] for _, v in items]
ax = fig.add_axes([0.30, 0.55, 0.60, 0.34])
y = range(len(names))
ax.barh(y, values, color=BLUE, height=0.55, zorder=3)
ax.set_yticks(list(y))
ax.set_yticklabels(names, fontsize=11)
ax.set_xlim(0, 1.0)
ax.grid(axis="x", color=GRID, linewidth=0.8, zorder=0)
ax.set_axisbelow(True)
for spine in ("top", "right"):
ax.spines[spine].set_visible(False)
ax.spines["left"].set_color("#0b0b0b")
ax.spines["bottom"].set_color("#0b0b0b")
ax.set_xlabel("mAP50", fontsize=10.5, color=MUTED)
for yi, (v, p) in enumerate(zip(values, pages)):
ax.text(v + 0.015, yi, f"{v:.3f} ({p} pages)", va="center", fontsize=9.5, fontweight="bold")
# failure notes, factual only
ax_title.text(0.07, 0.42, "Observed failure patterns", fontsize=13, fontweight="bold")
notes = [
"Footnote (47 instances): recall 0.048 — rare class, near-total miss rate.",
"Page-footer (408 instances): mAP50 0.843 — same thin shape as Footnote, but",
" 9x more training instances and a consistent position. Rarity, not size,",
" is the dominant factor.",
"Picture: precision 0.416 despite recall 0.622 — composite regions (diagrams",
" with embedded text) get split into multiple overlapping boxes instead",
" of one region.",
"Dense repeated-entry layouts (directories, org charts) produce duplicate,",
" overlapping box proposals rather than one box per entry.",
]
for i, line in enumerate(notes):
ax_title.text(0.07, 0.385 - i * 0.028, line, fontsize=10, color="#0b0b0b")
return fig
PER_CLASS_MAP50 = {
"Caption": 0.622, "Footnote": 0.182, "Formula": 0.788, "List-item": 0.671,
"Page-footer": 0.843, "Page-header": 0.658, "Picture": 0.495,
"Section-header": 0.762, "Table": 0.761, "Text": 0.832, "Title": 0.224,
}
if __name__ == "__main__":
import sys
preview = "--preview" in sys.argv
out_path = ROOT / "reports" / "evaluation_report.pdf"
pages = [page_overview(), page_per_class(), page_per_category()]
with PdfPages(out_path, metadata={
"Title": "", "Author": "", "Subject": "", "Creator": "", "Producer": "", "CreationDate": None,
}) as pdf:
for i, fig in enumerate(pages, start=1):
pdf.savefig(fig)
if preview:
fig.savefig(ROOT / f"reports/_preview_page{i}.png", dpi=110)
plt.close(fig)
print(f"wrote {out_path}")
|