feat: enhance training pipeline with multi-model evaluation and detailed metrics

app/training/train_classifier.py

@@ -1,16 +1,26 @@
 """
-Train AURIS classifier on extracted audio features.
+Comprehensive multi-model training pipeline for AURIS.
 
-Increment 1: RandomForest / GradientBoosting on librosa + vocal features.
-This replaces the heuristic scoring with a data-driven classifier.
+Trains and evaluates multiple classifier families on extracted
+audio features using stratified k-fold cross-validation, then
+selects the best model and exports it for production use.
+
+Models compared:
+  - Random Forest
+  - Gradient Boosting
+  - XGBoost
+  - LightGBM
+  - Support Vector Machine (RBF)
+  - Multi-Layer Perceptron (Neural Network)
 
 Usage:
-    python -m app.training.train_classifier data/sonics/features.csv
+    python -m app.training.train_classifier data/training/features.csv
 
 Outputs:
-    models/auris_classifier_v1.pkl   — trained model
+    models/auris_classifier_v1.pkl   — best trained model
     models/feature_scaler_v1.pkl     — fitted StandardScaler
     models/feature_columns_v1.json   — ordered feature column names
+    models/training_results.json     — all model metrics + CV folds
 """
 
 from __future__ import annotations
@@ -19,7 +29,9 @@ import csv
 import json
 import pickle
 import sys
+import time
 from pathlib import Path
+from typing import Any
 
 import numpy as np
 
@@ -27,6 +39,9 @@ from sklearn.ensemble import (
     GradientBoostingClassifier,
     RandomForestClassifier,
 )
+from sklearn.linear_model import LogisticRegression
+from sklearn.neural_network import MLPClassifier
+from sklearn.svm import SVC
 from sklearn.model_selection import (
     StratifiedKFold,
     cross_val_predict,
@@ -35,10 +50,19 @@ from sklearn.preprocessing import StandardScaler
 from sklearn.metrics import (
     accuracy_score,
     f1_score,
+    precision_score,
+    recall_score,
     roc_auc_score,
 )
 
-# Optional: LightGBM for better performance
+# Optional: XGBoost
+try:
+    import xgboost as xgb
+    HAS_XGB = True
+except ImportError:
+    HAS_XGB = False
+
+# Optional: LightGBM
 try:
     import lightgbm as lgb
     HAS_LGBM = True
@@ -56,15 +80,12 @@ def train(
     features_csv: str | Path,
     models_dir: str | Path = "models",
     n_folds: int = 5,
-) -> dict:
+) -> dict[str, Any]:
     """
-    Train and evaluate classifier on extracted features.
-
-    Uses 5-fold cross-validation to estimate real accuracy,
-    then trains final model on all data.
+    Train and evaluate all classifier candidates.
 
     Returns:
-        Dict with metrics and model paths.
+        Dict with per-model metrics, best model info, and saved paths.
     """
     models_dir = Path(models_dir)
     models_dir.mkdir(parents=True, exist_ok=True)
@@ -72,7 +93,6 @@ def train(
     # ── Load data ──────────────────────────────────
     X, y = load_features_csv(features_csv)
 
-    # Get feature column names
     with open(features_csv, "r", encoding="utf-8") as f:
         reader = csv.DictReader(f)
         feature_cols = [
@@ -87,39 +107,54 @@ def train(
     scaler = StandardScaler()
     X_scaled = scaler.fit_transform(X)
 
-    # ── Train multiple models, pick best ───────────
+    # ── Train multiple models ──────────────────────
     candidates = _build_candidates()
+    cv = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=42)
+
     best_model = None
     best_name = ""
     best_auc = 0.0
-    results = {}
-
-    cv = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=42)
+    all_results: dict[str, dict[str, Any]] = {}
 
     for name, model in candidates:
-        print(f"\n{'─' * 40}")
-        print(f"Training: {name}")
-        print(f"{'─' * 40}")
+        print(f"\n{'─' * 50}")
+        print(f"  Training: {name}")
+        print(f"{'─' * 50}")
 
-        # Cross-validated predictions
+        t0 = time.time()
+
+        # Cross-validated probability predictions
         y_prob = cross_val_predict(
             model, X_scaled, y,
             cv=cv, method="predict_proba",
         )[:, 1]
         y_pred = (y_prob > 0.5).astype(int)
 
+        train_time = time.time() - t0
+
         acc = accuracy_score(y, y_pred)
-        f1 = f1_score(y, y_pred)
+        prec = precision_score(y, y_pred, zero_division=0)
+        rec = recall_score(y, y_pred, zero_division=0)
+        f1 = f1_score(y, y_pred, zero_division=0)
         auc = roc_auc_score(y, y_prob)
 
-        print(f"  CV Accuracy: {acc:.4f}")
-        print(f"  CV F1:       {f1:.4f}")
-        print(f"  CV ROC-AUC:  {auc:.4f}")
+        print(f"  Accuracy:   {acc:.4f}")
+        print(f"  Precision:  {prec:.4f}")
+        print(f"  Recall:     {rec:.4f}")
+        print(f"  F1 Score:   {f1:.4f}")
+        print(f"  ROC-AUC:    {auc:.4f}")
+        print(f"  Train time: {train_time:.1f}s")
 
-        results[name] = {
+        all_results[name] = {
             "accuracy": round(acc, 4),
+            "precision": round(prec, 4),
+            "recall": round(rec, 4),
             "f1": round(f1, 4),
             "roc_auc": round(auc, 4),
+            "train_time_sec": round(train_time, 2),
+            "y_true": y.tolist(),
+            "y_pred": y_pred.tolist(),
+            "y_prob": y_prob.tolist(),
         }
 
         if auc > best_auc:
@@ -128,16 +163,12 @@ def train(
             best_model = model
 
     # ── Final evaluation of best model ─────────────
-    print(f"\n{'=' * 50}")
-    print(f"  Best model: {best_name} (AUC={best_auc:.4f})")
-    print(f"{'=' * 50}")
+    print(f"\n{'═' * 60}")
+    print(f"  BEST MODEL: {best_name}  (ROC-AUC = {best_auc:.4f})")
+    print(f"{'═' * 60}")
 
-    # Cross-val predictions for detailed report
-    y_prob_best = cross_val_predict(
-        best_model, X_scaled, y,
-        cv=cv, method="predict_proba",
-    )[:, 1]
-    y_pred_best = (y_prob_best > 0.5).astype(int)
+    y_prob_best = np.array(all_results[best_name]["y_prob"])
+    y_pred_best = np.array(all_results[best_name]["y_pred"])
 
     evaluate_predictions(
         y, y_pred_best, y_prob_best,
@@ -145,26 +176,22 @@ def train(
     )
 
     # ── Train final model on ALL data ──────────────
-    print(f"\nTraining final {best_name} on all data...")
+    print(f"\nTraining final {best_name} on all {len(y)} samples...")
     best_model.fit(X_scaled, y)
 
     # ── Feature importance ─────────────────────────
-    if hasattr(best_model, "feature_importances_"):
-        importances = best_model.feature_importances_
-        top_features = sorted(
-            zip(feature_cols, importances),
-            key=lambda x: x[1],
-            reverse=True,
-        )
-        print("\nTop 10 features:")
-        for fname, imp in top_features[:10]:
+    importance_data = _extract_importance(best_model, feature_cols)
+    if importance_data:
+        print("\nTop 15 features:")
+        for fname, imp in importance_data[:15]:
             bar = "█" * int(imp * 100)
-            print(f"  {fname:<30} {imp:.4f} {bar}")
+            print(f"  {fname:<35} {imp:.4f} {bar}")
 
     # ── Save artifacts ─────────────────────────────
     model_path = models_dir / "auris_classifier_v1.pkl"
     scaler_path = models_dir / "feature_scaler_v1.pkl"
     columns_path = models_dir / "feature_columns_v1.json"
+    results_path = models_dir / "training_results.json"
 
     with open(model_path, "wb") as f:
         pickle.dump(best_model, f)
@@ -173,24 +200,54 @@ def train(
     with open(columns_path, "w") as f:
         json.dump(feature_cols, f, indent=2)
 
+    # Save full results (without numpy arrays for JSON)
+    json_results = {}
+    for name, data in all_results.items():
+        json_results[name] = {
+            k: v for k, v in data.items()
+            if k not in ("y_true", "y_pred", "y_prob")
+        }
+    json_results["_best_model"] = best_name
+    json_results["_n_samples"] = len(y)
+    json_results["_n_features"] = X.shape[1]
+    json_results["_n_folds"] = n_folds
+    if importance_data:
+        json_results["_feature_importance"] = {
+            name: round(imp, 6) for name, imp in importance_data
+        }
+
+    with open(results_path, "w") as f:
+        json.dump(json_results, f, indent=2)
+
     print(f"\nSaved:")
     print(f"  Model:   {model_path}")
     print(f"  Scaler:  {scaler_path}")
     print(f"  Columns: {columns_path}")
+    print(f"  Results: {results_path}")
 
     return {
         "best_model": best_name,
         "best_auc": best_auc,
-        "results": results,
+        "all_results": all_results,
+        "feature_cols": feature_cols,
         "model_path": str(model_path),
     }
 
 
-def _build_candidates() -> list[tuple[str, object]]:
+def _build_candidates() -> list[tuple[str, Any]]:
     """Build list of classifier candidates to evaluate."""
-    candidates = [
+    candidates: list[tuple[str, Any]] = [
         (
-            "RandomForest",
+            "Logistic Regression",
+            LogisticRegression(
+                C=1.0,
+                max_iter=1000,
+                class_weight="balanced",
+                random_state=42,
+            ),
+        ),
+        (
+            "Random Forest",
             RandomForestClassifier(
                 n_estimators=300,
                 max_depth=20,
@@ -201,7 +258,7 @@ def _build_candidates() -> list[tuple[str, object]]:
             ),
         ),
         (
-            "GradientBoosting",
+            "Gradient Boosting",
             GradientBoostingClassifier(
                 n_estimators=200,
                 max_depth=6,
@@ -210,8 +267,49 @@ def _build_candidates() -> list[tuple[str, object]]:
                 random_state=42,
             ),
         ),
+        (
+            "SVM (RBF)",
+            SVC(
+                kernel="rbf",
+                C=10.0,
+                gamma="scale",
+                class_weight="balanced",
+                probability=True,
+                random_state=42,
+            ),
+        ),
+        (
+            "MLP Neural Network",
+            MLPClassifier(
+                hidden_layer_sizes=(128, 64, 32),
+                activation="relu",
+                solver="adam",
+                alpha=0.001,
+                learning_rate="adaptive",
+                max_iter=500,
+                early_stopping=True,
+                validation_fraction=0.15,
+                random_state=42,
+            ),
+        ),
     ]
 
+    if HAS_XGB:
+        candidates.append((
+            "XGBoost",
+            xgb.XGBClassifier(
+                n_estimators=300,
+                max_depth=8,
+                learning_rate=0.05,
+                subsample=0.8,
+                colsample_bytree=0.8,
+                scale_pos_weight=1.0,
+                eval_metric="logloss",
+                random_state=42,
+                verbosity=0,
+            ),
+        ))
+
     if HAS_LGBM:
         candidates.append((
             "LightGBM",
@@ -231,7 +329,34 @@ def _build_candidates() -> list[tuple[str, object]]:
     return candidates
 
 
+def _extract_importance(
+    model: Any,
+    feature_cols: list[str],
+) -> list[tuple[str, float]]:
+    """Extract feature importance from the trained model."""
+    importances = None
+
+    if hasattr(model, "feature_importances_"):
+        importances = model.feature_importances_
+    elif hasattr(model, "coef_"):
+        importances = np.abs(model.coef_[0])
+
+    if importances is None:
+        return []
+
+    # Normalize to sum to 1
+    total = np.sum(importances)
+    if total > 0:
+        importances = importances / total
+
+    return sorted(
+        zip(feature_cols, importances.tolist()),
+        key=lambda x: x[1],
+        reverse=True,
+    )
+
+
 if __name__ == "__main__":
-    csv_path = sys.argv[1] if len(sys.argv) > 1 else "data/sonics/features.csv"
+    csv_path = sys.argv[1] if len(sys.argv) > 1 else "data/training/features.csv"
     model_dir = sys.argv[2] if len(sys.argv) > 2 else "models"
     train(csv_path, model_dir)