feat: add deep-learning figure generation script for enhanced model evaluation

app/training/generate_deep_figures.py

@@ -0,0 +1,431 @@
+"""Deep-learning style publication figures.
+
+Produces additional academic figures beyond the 8 base charts:
+  - training_history.png       — XGBoost learning curve across boosting rounds
+  - per_class_metrics.png      — precision/recall/F1 per class (Human/AI)
+  - learning_curve.png         — train vs CV score vs training set size
+  - threshold_sweep.png        — precision/recall/F1 across thresholds
+  - score_distribution.png     — predicted-probability histogram by true class
+  - per_source_performance.png — breakdown by dataset source
+  - classification_report.png  — styled report table
+
+Usage:
+    python -m app.training.generate_deep_figures
+"""
+
+from __future__ import annotations
+
+import csv
+import json
+import pickle
+from pathlib import Path
+
+import numpy as np
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import matplotlib.patches as mpatches
+
+from sklearn.metrics import (
+    precision_recall_fscore_support,
+    precision_score, recall_score, f1_score,
+)
+from sklearn.model_selection import (
+    StratifiedKFold, cross_val_predict, learning_curve,
+)
+from sklearn.base import clone
+
+BACKEND = Path(__file__).resolve().parents[2]
+MODELS_DIR = BACKEND / "models"
+DATASET_DIR = BACKEND.parent / "DataSet"
+FIGURES_DIR = DATASET_DIR / "figures"
+FEATURES_CSV = DATASET_DIR / "features.csv"
+METADATA_CSV = DATASET_DIR / "metadata.csv"
+
+PALETTE = {
+    "bg": "#faf6ed",
+    "fg": "#3d2817",
+    "primary": "#c99347",
+    "secondary": "#7fb069",
+    "error": "#a64b3c",
+    "grid": "#d8c9a8",
+    "accent": "#e7c77a",
+    "human": "#7fb069",
+    "ai": "#a64b3c",
+}
+
+plt.rcParams.update({
+    "figure.facecolor": PALETTE["bg"],
+    "axes.facecolor": PALETTE["bg"],
+    "axes.edgecolor": PALETTE["fg"],
+    "axes.labelcolor": PALETTE["fg"],
+    "xtick.color": PALETTE["fg"],
+    "ytick.color": PALETTE["fg"],
+    "text.color": PALETTE["fg"],
+    "font.family": "DejaVu Sans",
+    "font.size": 11,
+    "axes.grid": True,
+    "grid.color": PALETTE["grid"],
+    "grid.alpha": 0.4,
+    "savefig.dpi": 150,
+    "savefig.bbox": "tight",
+    "figure.dpi": 100,
+})
+
+
+def _load():
+    with open(MODELS_DIR / "auris_classifier_v1.pkl", "rb") as f:
+        model = pickle.load(f)
+    with open(MODELS_DIR / "feature_scaler_v1.pkl", "rb") as f:
+        scaler = pickle.load(f)
+    with open(MODELS_DIR / "feature_columns_v1.json", "r") as f:
+        feature_cols = json.load(f)
+    with open(MODELS_DIR / "training_results.json", "r") as f:
+        results = json.load(f)
+    return model, scaler, feature_cols, results
+
+
+def _load_data(feature_cols):
+    with open(FEATURES_CSV, "r", encoding="utf-8") as f:
+        rows = list(csv.DictReader(f))
+    X = np.array([[float(r[c]) for c in feature_cols] for r in rows])
+    X = np.nan_to_num(X, nan=0.0, posinf=1.0, neginf=-1.0)
+    y = np.array([int(r["label_int"]) for r in rows])
+    paths = [r.get("path", "") for r in rows]
+    return X, y, paths, rows
+
+
+def _cv_predict(model, X_scaled, y):
+    cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
+    y_prob = cross_val_predict(
+        clone(model), X_scaled, y, cv=cv, method="predict_proba", n_jobs=-1,
+    )[:, 1]
+    y_pred = (y_prob > 0.5).astype(int)
+    return y_pred, y_prob
+
+
+# ── 1. Training history (XGBoost boosting-round learning curve) ──────────
+def fig_training_history(model, scaler, X, y):
+    """Retrain lightly with eval_set to capture boosting progression."""
+    from xgboost import XGBClassifier
+    from sklearn.model_selection import train_test_split
+
+    X_scaled = scaler.transform(X)
+    X_tr, X_val, y_tr, y_val = train_test_split(
+        X_scaled, y, test_size=0.2, stratify=y, random_state=42,
+    )
+
+    params = model.get_params()
+    # Reset early-stopping / n_estimators for a fresh fit with eval tracking
+    params["n_estimators"] = min(params.get("n_estimators", 300) or 300, 500)
+    params["eval_metric"] = ["logloss", "error", "auc"]
+
+    clf = XGBClassifier(**{k: v for k, v in params.items() if k != "early_stopping_rounds"})
+    clf.fit(
+        X_tr, y_tr,
+        eval_set=[(X_tr, y_tr), (X_val, y_val)],
+        verbose=False,
+    )
+
+    history = clf.evals_result()
+    tr = history["validation_0"]
+    vl = history["validation_1"]
+
+    fig, axes = plt.subplots(1, 3, figsize=(16, 5))
+    x = np.arange(1, len(tr["logloss"]) + 1)
+
+    for ax, metric, title in [
+        (axes[0], "logloss", "Log Loss"),
+        (axes[1], "error", "Error Rate"),
+        (axes[2], "auc", "ROC-AUC"),
+    ]:
+        ax.plot(x, tr[metric], color=PALETTE["primary"], lw=2.2, label="Eğitim / Train")
+        ax.plot(x, vl[metric], color=PALETTE["error"], lw=2.2,
+                linestyle="--", label="Doğrulama / Validation")
+        ax.set_xlabel("Boosting Round")
+        ax.set_ylabel(title)
+        ax.set_title(f"{title} — Boosting İlerlemesi", fontweight="bold")
+        ax.legend(framealpha=0.85)
+        # best round annotation
+        best_idx = int(np.argmin(vl["logloss"])) if metric == "logloss" else int(np.argmax(vl[metric]))
+        ax.axvline(best_idx + 1, color=PALETTE["accent"], linestyle=":", alpha=0.7)
+        ax.annotate(
+            f"en iyi: {best_idx + 1}",
+            xy=(best_idx + 1, vl[metric][best_idx]),
+            xytext=(12, -12), textcoords="offset points",
+            fontsize=9, color=PALETTE["fg"],
+        )
+
+    fig.suptitle("XGBoost Eğitim Geçmişi — Train vs Validation", fontsize=14, fontweight="bold")
+    plt.tight_layout()
+    plt.savefig(FIGURES_DIR / "training_history.png")
+    plt.close()
+    print("  ✓ training_history.png")
+
+
+# ── 2. Per-class precision/recall/F1 ─────────────────────────────────────
+def fig_per_class_metrics(y_true, y_pred):
+    p, r, f, support = precision_recall_fscore_support(y_true, y_pred)
+    classes = ["İnsan / Human", "AI / Yapay"]
+    metrics = {"Precision": p, "Recall": r, "F1 Score": f}
+
+    fig, ax = plt.subplots(figsize=(9, 6))
+    x = np.arange(len(classes))
+    width = 0.25
+    colors = [PALETTE["primary"], PALETTE["secondary"], PALETTE["error"]]
+
+    for i, (name, vals) in enumerate(metrics.items()):
+        bars = ax.bar(x + (i - 1) * width, vals, width, label=name,
+                      color=colors[i], edgecolor=PALETTE["fg"], linewidth=0.5)
+        for bar, v in zip(bars, vals):
+            ax.text(bar.get_x() + bar.get_width() / 2, v + 0.01,
+                    f"{v:.3f}", ha="center", va="bottom", fontsize=10, fontweight="bold")
+
+    ax.set_xticks(x)
+    ax.set_xticklabels([f"{c}\n(n={s})" for c, s in zip(classes, support)])
+    ax.set_ylabel("Skor / Score")
+    ax.set_title("Sınıf Başına Performans — Precision / Recall / F1",
+                 fontsize=13, fontweight="bold")
+    ax.set_ylim([0, 1.08])
+    ax.legend(loc="lower right", framealpha=0.85)
+    plt.savefig(FIGURES_DIR / "per_class_metrics.png")
+    plt.close()
+    print("  ✓ per_class_metrics.png")
+
+
+# ── 3. Learning curve (score vs training set size) ───────────────────────
+def fig_learning_curve(model, scaler, X, y):
+    X_scaled = scaler.transform(X)
+    train_sizes = np.linspace(0.1, 1.0, 6)
+    cv = StratifiedKFold(n_splits=3, shuffle=True, random_state=42)
+
+    sizes, tr_scores, val_scores = learning_curve(
+        clone(model), X_scaled, y,
+        train_sizes=train_sizes, cv=cv,
+        scoring="roc_auc", n_jobs=-1,
+        random_state=42,
+    )
+    tr_mean, tr_std = tr_scores.mean(1), tr_scores.std(1)
+    val_mean, val_std = val_scores.mean(1), val_scores.std(1)
+
+    fig, ax = plt.subplots(figsize=(9, 6.5))
+    ax.plot(sizes, tr_mean, "o-", color=PALETTE["primary"], lw=2.5, label="Eğitim / Train")
+    ax.fill_between(sizes, tr_mean - tr_std, tr_mean + tr_std,
+                    alpha=0.18, color=PALETTE["primary"])
+    ax.plot(sizes, val_mean, "s-", color=PALETTE["error"], lw=2.5,
+            label="Çapraz Doğrulama / Cross-Validation")
+    ax.fill_between(sizes, val_mean - val_std, val_mean + val_std,
+                    alpha=0.18, color=PALETTE["error"])
+
+    ax.set_xlabel("Eğitim Örneği Sayısı / Training Examples")
+    ax.set_ylabel("ROC-AUC")
+    ax.set_title("Öğrenme Eğrisi — Model Veri ile Öğreniyor mu?",
+                 fontsize=13, fontweight="bold")
+    ax.legend(loc="lower right", framealpha=0.85)
+    gap = tr_mean[-1] - val_mean[-1]
+    diagnosis = "düşük varyans (iyi)" if gap < 0.03 else "overfitting işareti"
+    ax.annotate(
+        f"Son gap: {gap:.4f}\n→ {diagnosis}",
+        xy=(0.55, 0.05), xycoords="axes fraction",
+        fontsize=10,
+        bbox=dict(boxstyle="round,pad=0.5", facecolor=PALETTE["bg"],
+                  edgecolor=PALETTE["primary"], alpha=0.85),
+    )
+    plt.savefig(FIGURES_DIR / "learning_curve.png")
+    plt.close()
+    print("  ✓ learning_curve.png")
+
+
+# ── 4. Threshold sweep ───────────────────────────────────────────────────
+def fig_threshold_sweep(y_true, y_prob):
+    thresholds = np.linspace(0.05, 0.95, 91)
+    precisions, recalls, f1s = [], [], []
+    for t in thresholds:
+        pred = (y_prob > t).astype(int)
+        precisions.append(precision_score(y_true, pred, zero_division=0))
+        recalls.append(recall_score(y_true, pred, zero_division=0))
+        f1s.append(f1_score(y_true, pred, zero_division=0))
+
+    precisions, recalls, f1s = np.array(precisions), np.array(recalls), np.array(f1s)
+    best_idx = int(np.argmax(f1s))
+    best_t = thresholds[best_idx]
+
+    fig, ax = plt.subplots(figsize=(10, 6))
+    ax.plot(thresholds, precisions, color=PALETTE["primary"], lw=2.5, label="Precision")
+    ax.plot(thresholds, recalls, color=PALETTE["secondary"], lw=2.5, label="Recall")
+    ax.plot(thresholds, f1s, color=PALETTE["error"], lw=2.8, label="F1 Score")
+    ax.axvline(0.5, color=PALETTE["fg"], linestyle=":", alpha=0.5, label="Varsayılan 0.5")
+    ax.axvline(best_t, color=PALETTE["accent"], linestyle="--", lw=2,
+               label=f"En iyi F1 @ {best_t:.2f}")
+    ax.scatter([best_t], [f1s[best_idx]], color=PALETTE["accent"],
+               s=100, zorder=5, edgecolor=PALETTE["fg"])
+
+    ax.set_xlabel("Karar Eşiği / Decision Threshold")
+    ax.set_ylabel("Skor")
+    ax.set_title("Eşik Taraması — Precision / Recall / F1 vs Threshold",
+                 fontsize=13, fontweight="bold")
+    ax.legend(loc="lower left", framealpha=0.85)
+    ax.set_xlim([0, 1])
+    ax.set_ylim([0, 1.02])
+    plt.savefig(FIGURES_DIR / "threshold_sweep.png")
+    plt.close()
+    print("  ✓ threshold_sweep.png")
+
+
+# ── 5. Score distribution histogram ──────────────────────────────────────
+def fig_score_distribution(y_true, y_prob):
+    fig, ax = plt.subplots(figsize=(10, 6))
+    bins = np.linspace(0, 1, 41)
+    human_probs = y_prob[y_true == 0]
+    ai_probs = y_prob[y_true == 1]
+
+    ax.hist(human_probs, bins=bins, alpha=0.65, color=PALETTE["human"],
+            label=f"İnsan (n={len(human_probs)})", edgecolor=PALETTE["fg"], linewidth=0.3)
+    ax.hist(ai_probs, bins=bins, alpha=0.65, color=PALETTE["ai"],
+            label=f"AI (n={len(ai_probs)})", edgecolor=PALETTE["fg"], linewidth=0.3)
+    ax.axvline(0.5, color=PALETTE["fg"], linestyle="--", alpha=0.7, lw=2,
+               label="Karar Eşiği")
+
+    ax.set_xlabel("Tahmin Olasılığı P(AI) / Predicted Probability")
+    ax.set_ylabel("Örnek Sayısı / Count")
+    ax.set_title("Tahmin Olasılığı Dağılımı — Sınıf Bazlı",
+                 fontsize=13, fontweight="bold")
+    ax.legend(framealpha=0.85)
+    plt.savefig(FIGURES_DIR / "score_distribution.png")
+    plt.close()
+    print("  ✓ score_distribution.png")
+
+
+# ── 6. Per-source breakdown ──────────────────────────────────────────────
+def fig_per_source_performance(y_true, y_pred, paths, rows):
+    # Join features.csv by path with metadata.csv source info
+    if not METADATA_CSV.exists():
+        print("  ! metadata.csv missing, skipping per_source_performance")
+        return
+
+    with open(METADATA_CSV, "r", encoding="utf-8") as f:
+        meta_rows = list(csv.DictReader(f))
+    # normalize paths for join (forward slashes)
+    path_to_source = {
+        r["path"].replace("\\", "/"): r.get("source", "unknown")
+        for r in meta_rows
+    }
+
+    sources_hits: dict[str, dict[str, int]] = {}
+    for yt, yp, path in zip(y_true, y_pred, paths):
+        key = path.replace("\\", "/")
+        src = path_to_source.get(key, "unknown")
+        d = sources_hits.setdefault(src, {"total": 0, "correct": 0, "ai": 0, "human": 0})
+        d["total"] += 1
+        if yt == yp:
+            d["correct"] += 1
+        d["ai" if yt == 1 else "human"] += 1
+
+    sources = [s for s in sources_hits if sources_hits[s]["total"] >= 20]
+    sources.sort(key=lambda s: -sources_hits[s]["total"])
+    if not sources:
+        print("  ! no source has >=20 samples, skipping")
+        return
+
+    accs = [sources_hits[s]["correct"] / sources_hits[s]["total"] for s in sources]
+    totals = [sources_hits[s]["total"] for s in sources]
+
+    fig, ax = plt.subplots(figsize=(10, max(4, len(sources) * 0.45)))
+    y_pos = np.arange(len(sources))
+    colors = plt.cm.copper(np.linspace(0.3, 0.9, len(sources)))
+    ax.barh(y_pos, accs, color=colors, edgecolor=PALETTE["fg"], linewidth=0.5)
+    ax.set_yticks(y_pos)
+    ax.set_yticklabels([f"{s} (n={n})" for s, n in zip(sources, totals)])
+    ax.invert_yaxis()
+    ax.set_xlabel("Accuracy")
+    ax.set_title("Veri Kaynağı Bazlı Performans",
+                 fontsize=13, fontweight="bold")
+    ax.set_xlim([0, 1.0])
+    for i, v in enumerate(accs):
+        ax.text(v + 0.005, i, f"{v:.3f}", va="center", fontsize=9)
+    plt.savefig(FIGURES_DIR / "per_source_performance.png")
+    plt.close()
+    print("  ✓ per_source_performance.png")
+
+
+# ── 7. Classification report as styled table ─────────────────────────────
+def fig_classification_report(y_true, y_pred):
+    from sklearn.metrics import classification_report
+    report = classification_report(
+        y_true, y_pred, target_names=["Human (İnsan)", "AI (Yapay)"],
+        digits=4, output_dict=True,
+    )
+
+    fig, ax = plt.subplots(figsize=(10, 4.5))
+    ax.axis("off")
+
+    classes = ["Human (İnsan)", "AI (Yapay)", "accuracy", "macro avg", "weighted avg"]
+    header = ["Class", "Precision", "Recall", "F1", "Support"]
+    data = [header]
+    for c in classes:
+        r = report.get(c, {})
+        if c == "accuracy":
+            data.append([c, "", "", f"{report['accuracy']:.4f}", f"{len(y_true)}"])
+        else:
+            data.append([
+                c,
+                f"{r.get('precision', 0):.4f}",
+                f"{r.get('recall', 0):.4f}",
+                f"{r.get('f1-score', 0):.4f}",
+                f"{int(r.get('support', 0))}",
+            ])
+
+    table = ax.table(
+        cellText=data, cellLoc="center", loc="center",
+        colWidths=[0.25, 0.18, 0.18, 0.18, 0.18],
+    )
+    table.auto_set_font_size(False)
+    table.set_fontsize(11)
+    table.scale(1, 1.8)
+
+    # header styling
+    for i in range(len(header)):
+        table[(0, i)].set_facecolor(PALETTE["primary"])
+        table[(0, i)].set_text_props(weight="bold", color=PALETTE["bg"])
+    # row stripes
+    for r in range(1, len(data)):
+        for c in range(len(header)):
+            table[(r, c)].set_facecolor(
+                PALETTE["bg"] if r % 2 else "#f0e6d0",
+            )
+            table[(r, c)].set_edgecolor(PALETTE["grid"])
+
+    ax.set_title("Sınıflandırma Raporu — 5-fold Cross-Validation",
+                 fontsize=13, fontweight="bold", pad=18)
+    plt.savefig(FIGURES_DIR / "classification_report.png")
+    plt.close()
+    print("  ✓ classification_report.png")
+
+
+def main():
+    FIGURES_DIR.mkdir(parents=True, exist_ok=True)
+    print(f"Output: {FIGURES_DIR}")
+    print("Loading...")
+    model, scaler, feature_cols, results = _load()
+    X, y, paths, rows = _load_data(feature_cols)
+
+    print("CV predictions (5-fold)...")
+    X_scaled = scaler.transform(X)
+    y_pred, y_prob = _cv_predict(model, X_scaled, y)
+
+    print("\nGenerating deep figures...")
+    fig_per_class_metrics(y, y_pred)
+    fig_threshold_sweep(y, y_prob)
+    fig_score_distribution(y, y_prob)
+    fig_per_source_performance(y, y_pred, paths, rows)
+    fig_classification_report(y, y_pred)
+    fig_training_history(model, scaler, X, y)
+    print("Learning curve (may take ~30s)...")
+    fig_learning_curve(model, scaler, X, y)
+
+    total = len(list(FIGURES_DIR.glob("*.png")))
+    print(f"\nDone. Total figures in {FIGURES_DIR}: {total}")
+
+
+if __name__ == "__main__":
+    main()