Create prototype_5_geodesic_prelim.py

prototype_5_geodesic_prelim.py

@@ -0,0 +1,855 @@
+# ============================================================================
+# RAPID PROTOTYPE v2: Differentiation-Centered Alignment Bank
+#
+# The bank aligns to the DIFFERENTIATION between experts, not to any
+# arbitrary target. The consensus CV, spectral profile, and pairwise
+# statistics measured during alignment become the exact targets.
+#
+# The bank embodies the centerpoint of expert disagreement.
+# ============================================================================
+
+import gc
+import math
+import os
+import time
+import json
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from tqdm import tqdm
+
+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+
+EXPERTS = [
+    ("google-bert/bert-base-uncased", "bert", 512),
+    ("answerdotai/ModernBERT-base", "modern", 512),
+]
+
+print("=" * 65)
+print("RAPID PROTOTYPE v2: Differentiation-Centered Bank")
+print("=" * 65)
+print(f"  Device: {DEVICE}")
+
+
+# ══════════════════════════════════════════════════════════════════
+# STUDENT MODEL
+# ══════════════════════════════════════════════════════════════════
+
+class MiniStudent(nn.Module):
+    def __init__(self, vocab_size=30522, max_len=512, d_model=256,
+                 n_heads=4, n_layers=4, d_ff=1024, output_dim=768,
+                 dropout=0.1, pad_token_id=0):
+        super().__init__()
+        self.pad_token_id = pad_token_id
+        self.token_emb = nn.Embedding(vocab_size, d_model, padding_idx=pad_token_id)
+        self.pos_emb = nn.Embedding(max_len, d_model)
+        self.emb_norm = nn.LayerNorm(d_model)
+        self.emb_drop = nn.Dropout(dropout)
+        encoder_layer = nn.TransformerEncoderLayer(
+            d_model=d_model, nhead=n_heads, dim_feedforward=d_ff,
+            dropout=dropout, activation="gelu", batch_first=True,
+            norm_first=True)
+        self.encoder = nn.TransformerEncoder(
+            encoder_layer, num_layers=n_layers, enable_nested_tensor=False)
+        self.output_proj = nn.Sequential(
+            nn.Linear(d_model, d_model), nn.GELU(),
+            nn.LayerNorm(d_model), nn.Linear(d_model, output_dim))
+
+    def forward(self, input_ids, attention_mask=None):
+        B, L = input_ids.shape
+        positions = torch.arange(L, device=input_ids.device).unsqueeze(0)
+        x = self.token_emb(input_ids) + self.pos_emb(positions)
+        x = self.emb_drop(self.emb_norm(x))
+        kpm = ~attention_mask.bool() if attention_mask is not None else (input_ids == self.pad_token_id)
+        x = self.encoder(x, src_key_padding_mask=kpm)
+        mask = attention_mask.unsqueeze(-1).float() if attention_mask is not None else (~kpm).unsqueeze(-1).float()
+        pooled = (x * mask).sum(1) / mask.sum(1).clamp(min=1)
+        return F.normalize(self.output_proj(pooled), dim=-1)
+
+
+# ══════════════════════════════════════════════════════════════════
+# ALIGNMENT BANK
+# ══════════════════════════════════════════════════════════════════
+
+class AlignmentBank(nn.Module):
+    """
+    Differentiation-centered geometric interface.
+
+    Aligns to the CENTERPOINT between experts — the consensus itself.
+    Stores per-expert rotation matrices (the differentiation structure)
+    and learned anchor landmarks (the consensus manifold topology).
+
+    The bank doesn't invent geometry. It mirrors the measured consensus.
+    Every loss term pulls toward measured consensus statistics.
+    """
+    def __init__(self, d_embed=768, n_experts=2, n_anchors=512, d_bank=128):
+        super().__init__()
+        self.d_embed = d_embed
+        self.n_experts = n_experts
+        self.n_anchors = n_anchors
+        self.d_bank = d_bank
+
+        # Per-expert rotation matrices (differentiation structure)
+        self.expert_rotations = nn.ParameterList([
+            nn.Parameter(torch.eye(d_embed)) for _ in range(n_experts)
+        ])
+
+        # Per-expert whiteners (captures variance structure per expert)
+        self.expert_whiteners = nn.ParameterList([
+            nn.Parameter(torch.eye(d_embed)) for _ in range(n_experts)
+        ])
+
+        # Per-expert means (centering offset per expert)
+        self.expert_means = nn.ParameterList([
+            nn.Parameter(torch.zeros(d_embed)) for _ in range(n_experts)
+        ])
+
+        # Anchor bank: consensus landmarks on the hypersphere
+        self.anchors = nn.Parameter(
+            F.normalize(torch.randn(n_anchors, d_embed), dim=-1))
+
+        # Project: expert_cos (n) + expert_mse (n) + cross (n*(n-1)/2) +
+        #          disagreement_ratio (1) + norm_ratio (n) + anchor_cos (n_anchors)
+        n_cross = n_experts * (n_experts - 1) // 2
+        geo_dim = n_experts + n_experts + n_cross + 1 + n_experts + n_anchors
+        self.geo_proj = nn.Sequential(
+            nn.Linear(geo_dim, d_bank * 2),
+            nn.GELU(),
+            nn.LayerNorm(d_bank * 2),
+            nn.Linear(d_bank * 2, d_bank),
+            nn.LayerNorm(d_bank),
+        )
+
+        # Consensus statistics (set during init, used as exact targets)
+        self.register_buffer("target_cv", torch.tensor(0.12))
+        self.register_buffer("target_mean_cos", torch.tensor(0.0))
+        self.register_buffer("target_spectral", torch.zeros(50))
+        # Disagreement structure (measured once, preserved forever)
+        self.register_buffer("target_cross_cos_mean", torch.tensor(0.0))
+        self.register_buffer("target_cross_cos_std", torch.tensor(0.0))
+        self.register_buffer("target_disagreement_ratio", torch.tensor(0.0))
+
+    def init_from_procrustes(self, procrustes_results, expert_names,
+                              consensus_embeddings=None,
+                              consensus_stats=None):
+        """Initialize from consensus training artifacts."""
+        device = self.anchors.device
+        for i, name in enumerate(expert_names[:self.n_experts]):
+            info = procrustes_results[name]
+            self.expert_rotations[i].data = info["rotation"].float().to(device)
+            if "source_whitener" in info:
+                self.expert_whiteners[i].data = info["source_whitener"].float().to(device)
+            if "source_mean" in info:
+                self.expert_means[i].data = info["source_mean"].float().to(device)
+            print(f"    Expert {i} ({name}): rotation + whitener + mean loaded, "
+                  f"cos_after={info['cos_after']:.4f}")
+
+        if consensus_embeddings is not None:
+            n = min(self.n_anchors, consensus_embeddings.shape[0])
+            indices = torch.linspace(0, consensus_embeddings.shape[0] - 1, n).long()
+            self.anchors.data[:n] = F.normalize(
+                consensus_embeddings[indices].float(), dim=-1).to(device)
+            print(f"    Anchors: {n} initialized from consensus embeddings")
+
+        if consensus_stats is not None:
+            self.target_cv.fill_(consensus_stats["cv"])
+            self.target_mean_cos.fill_(consensus_stats["mean_cos"])
+            if "spectral" in consensus_stats:
+                s = torch.tensor(consensus_stats["spectral"][:50], dtype=torch.float32)
+                self.target_spectral[:len(s)] = s.to(device)
+            print(f"    Targets: CV={consensus_stats['cv']:.4f}, "
+                  f"mean_cos={consensus_stats['mean_cos']:.4f}")
+
+    def forward(self, embedding):
+        B = embedding.shape[0]
+        emb = embedding.float()
+
+        # ── Per-expert projections (full whitened Procrustes) ──
+        # Chain: center → whiten → normalize → rotate
+        # This is EXACTLY what was computed during alignment.
+        # The rotation only makes geometric sense in whitened-normalized space.
+        expert_consistency = []
+        expert_recon = []
+        expert_projected = []
+        for i in range(self.n_experts):
+            R = self.expert_rotations[i]
+            W = self.expert_whiteners[i]
+            mu = self.expert_means[i]
+
+            # Forward: center → whiten → normalize → rotate
+            centered = emb - mu
+            whitened = centered @ W
+            whitened_n = F.normalize(whitened, dim=-1)
+            in_expert = whitened_n @ R.T  # now in expert's whitened-normalized space
+
+            # Round-trip: rotate back (orthogonal, so R.T inverse = R)
+            back = in_expert @ R
+
+            # Consistency: round-trip should recover whitened_n exactly
+            cos = F.cosine_similarity(whitened_n, back, dim=-1)
+            recon = (whitened_n - back).pow(2).mean(dim=-1)
+
+            expert_consistency.append(cos)
+            expert_recon.append(recon)
+            expert_projected.append(in_expert)
+
+        expert_cos = torch.stack(expert_consistency, dim=-1)   # (B, n_experts)
+        expert_mse = torch.stack(expert_recon, dim=-1)         # (B, n_experts)
+
+        # ── Cross-expert differentiation ──
+        # How each expert's projection relates to every other expert's projection
+        # This IS the disagreement structure. Preserve it exactly.
+        cross_cos = []
+        for i in range(self.n_experts):
+            for j in range(i + 1, self.n_experts):
+                cc = F.cosine_similarity(
+                    expert_projected[i], expert_projected[j], dim=-1)
+                cross_cos.append(cc)
+        cross_features = torch.stack(cross_cos, dim=-1) if cross_cos else torch.zeros(B, 0, device=emb.device)
+
+        # Per-sample disagreement: how much do experts disagree on THIS embedding?
+        # High disagreement = embedding is in contested territory
+        # Low disagreement = all experts agree (well-anchored)
+        per_sample_agreement = expert_cos.mean(dim=-1)           # (B,) mean round-trip cos
+        per_sample_disagreement = expert_cos.std(dim=-1)         # (B,) std across experts
+        # Ratio: how much agreement relative to disagreement
+        disagreement_ratio = per_sample_disagreement / (per_sample_agreement + 1e-8)  # (B,)
+
+        # Expert projection norms before normalization (captures magnitude structure)
+        expert_norms = []
+        for i in range(self.n_experts):
+            R = self.expert_rotations[i]
+            W = self.expert_whiteners[i]
+            mu = self.expert_means[i]
+            centered = emb - mu
+            whitened = centered @ W
+            expert_norms.append(whitened.norm(dim=-1))  # (B,)
+        expert_norm_features = torch.stack(expert_norms, dim=-1)  # (B, n_experts)
+        norm_ratio = expert_norm_features / (expert_norm_features.mean(dim=-1, keepdim=True) + 1e-8)
+
+        # ── Anchor distances ──
+        anchors_n = F.normalize(self.anchors, dim=-1)
+        anchor_cos = emb @ anchors_n.T  # (B, n_anchors)
+
+        # ── Geometric context ──
+        # Full feature set: expert consistency + reconstruction + cross-expert +
+        #                   disagreement ratio + norm ratios + anchor distances
+        geo_input = torch.cat([
+            expert_cos,                                 # (B, n_experts)
+            expert_mse,                                 # (B, n_experts)
+            cross_features,                             # (B, n_cross)
+            disagreement_ratio.unsqueeze(-1),            # (B, 1)
+            norm_ratio,                                 # (B, n_experts)
+            anchor_cos,                                 # (B, n_anchors)
+        ], dim=-1)
+        geo_context = self.geo_proj(geo_input)
+
+        enriched = torch.cat([embedding, geo_context], dim=-1)
+
+        # ── Losses + Diagnostics ──
+        aux = {}
+
+        # 1. Expert agreement: all experts should see embedding equally
+        expert_mean = expert_cos.mean(dim=-1, keepdim=True)
+        aux["expert_agreement"] = (expert_cos - expert_mean).pow(2).mean()
+
+        # 2. Rotation orthogonality
+        ortho_loss = 0.0
+        for i in range(self.n_experts):
+            R = self.expert_rotations[i]
+            RRT = R @ R.T
+            ortho_loss += (RRT - torch.eye(self.d_embed, device=R.device)).pow(2).mean()
+        aux["rotation_ortho"] = ortho_loss / self.n_experts
+
+        # 3. Anchor spread
+        anchor_sim = anchors_n @ anchors_n.T
+        anchor_sim.fill_diagonal_(0)
+        aux["anchor_spread"] = anchor_sim.pow(2).mean()
+
+        # 4. Anchor sharpness
+        anchor_probs = F.softmax(anchor_cos * 10, dim=-1)
+        entropy = -(anchor_probs * (anchor_probs + 1e-12).log()).sum(-1).mean()
+        aux["anchor_entropy"] = entropy
+
+        # 5. Cross-expert differentiation consistency
+        if cross_features.shape[1] > 0:
+            aux["cross_expert_var"] = cross_features.var(dim=0).mean()
+        else:
+            aux["cross_expert_var"] = torch.tensor(0.0, device=emb.device)
+
+        # 6. Disagreement preservation
+        # The distribution of disagreement should stay at the measured target
+        batch_cross_mean = cross_features.mean() if cross_features.shape[1] > 0 else torch.tensor(0.0, device=emb.device)
+        batch_cross_std = cross_features.std() if cross_features.shape[1] > 0 else torch.tensor(0.0, device=emb.device)
+        batch_disagree_ratio = disagreement_ratio.mean()
+        aux["disagree_preserve"] = (
+            (batch_cross_mean - self.target_cross_cos_mean).pow(2) +
+            (batch_cross_std - self.target_cross_cos_std).pow(2) +
+            (batch_disagree_ratio - self.target_disagreement_ratio).pow(2)
+        )
+
+        # 7. Bank CV
+        if B >= 10:
+            ctx_n = F.normalize(geo_context, dim=-1)
+            vols = []
+            for _ in range(32):
+                idx = torch.randperm(B, device=embedding.device)[:5]
+                pts = ctx_n[idx].unsqueeze(0)
+                diff = pts.unsqueeze(-2) - pts.unsqueeze(-3)
+                d2 = (diff * diff).sum(-1)
+                Bv, V, _ = d2.shape
+                cm = torch.zeros(Bv, V+1, V+1, device=d2.device, dtype=torch.float32)
+                cm[:, 0, 1:] = 1; cm[:, 1:, 0] = 1; cm[:, 1:, 1:] = d2
+                s = (-1.0)**V; f = math.factorial(V-1)
+                v2 = s / ((2.0**(V-1)) * f*f) * torch.linalg.det(cm)
+                vols.append(torch.sqrt(F.relu(v2[0]) + 1e-12))
+            stacked = torch.stack(vols)
+            bank_cv = stacked.std() / (stacked.mean() + 1e-8)
+            aux["bank_cv"] = bank_cv
+        else:
+            aux["bank_cv"] = torch.tensor(0.0, device=embedding.device)
+
+        # 8. Emb CV
+        if B >= 10:
+            emb_n = F.normalize(emb, dim=-1)
+            vols = []
+            for _ in range(32):
+                idx = torch.randperm(B, device=embedding.device)[:5]
+                pts = emb_n[idx].unsqueeze(0)
+                diff = pts.unsqueeze(-2) - pts.unsqueeze(-3)
+                d2 = (diff * diff).sum(-1)
+                Bv, V, _ = d2.shape
+                cm = torch.zeros(Bv, V+1, V+1, device=d2.device, dtype=torch.float32)
+                cm[:, 0, 1:] = 1; cm[:, 1:, 0] = 1; cm[:, 1:, 1:] = d2
+                s = (-1.0)**V; f = math.factorial(V-1)
+                v2 = s / ((2.0**(V-1)) * f*f) * torch.linalg.det(cm)
+                vols.append(torch.sqrt(F.relu(v2[0]) + 1e-12))
+            stacked = torch.stack(vols)
+            emb_cv = stacked.std() / (stacked.mean() + 1e-8)
+            aux["emb_cv"] = emb_cv
+        else:
+            aux["emb_cv"] = torch.tensor(0.0, device=embedding.device)
+
+        # Diagnostics
+        aux["expert_cos_mean"] = expert_cos.mean().item()
+        aux["expert_cos_std"] = expert_cos.std().item()
+        aux["anchor_max_cos"] = anchor_cos.max(dim=-1).values.mean().item()
+        aux["anchor_mean_cos"] = anchor_cos.mean().item()
+        if cross_features.shape[1] > 0:
+            aux["cross_expert_cos"] = cross_features.mean().item()
+            aux["cross_expert_cos_std"] = cross_features.std().item()
+        aux["disagreement_ratio"] = disagreement_ratio.mean().item()
+        aux["norm_ratio_spread"] = norm_ratio.std(dim=-1).mean().item()
+
+        return enriched, aux
+
+    def bank_loss(self, aux):
+        """All targets from measured consensus. Preserves disagreement structure."""
+        loss = (
+            1.0 * aux["expert_agreement"] +
+            1.0 * aux["rotation_ortho"] +
+            0.5 * aux["anchor_spread"] +
+            0.1 * aux["anchor_entropy"] +
+            0.3 * aux["cross_expert_var"] +
+            0.3 * (aux["bank_cv"] - self.target_cv).abs() +
+            0.3 * (aux["emb_cv"] - self.target_cv).abs() +
+            0.5 * aux["disagree_preserve"]  # preserve the disagreement distribution
+        )
+        return loss
+
+    @torch.no_grad()
+    def calibrate_disagreement(self, embeddings):
+        """
+        Measure the initial disagreement structure and store as targets.
+        Call ONCE after init, before training.
+        """
+        _, aux = self.forward(embeddings)
+        if "cross_expert_cos" in aux:
+            self.target_cross_cos_mean.fill_(aux["cross_expert_cos"])
+        if "cross_expert_cos_std" in aux:
+            self.target_cross_cos_std.fill_(aux["cross_expert_cos_std"])
+        self.target_disagreement_ratio.fill_(aux["disagreement_ratio"])
+        print(f"    Calibrated disagreement:")
+        print(f"      cross_cos: {self.target_cross_cos_mean.item():.4f} ± {self.target_cross_cos_std.item():.4f}")
+        print(f"      disagree_ratio: {self.target_disagreement_ratio.item():.6f}")
+
+
+# ══════════════════════════════════════════════════════════════════
+# GEOMETRY
+# ══════════════════════════════════════════════════════════════════
+
+def infonce(a, b, temperature=0.07):
+    a = F.normalize(a, dim=-1)
+    b = F.normalize(b, dim=-1)
+    logits = (a @ b.T) / temperature
+    labels = torch.arange(logits.shape[0], device=logits.device)
+    loss = (F.cross_entropy(logits, labels) + F.cross_entropy(logits.T, labels)) / 2
+    with torch.no_grad():
+        acc = (logits.argmax(-1) == labels).float().mean().item()
+    return loss, acc
+
+def cayley_menger_vol2(pts):
+    pts = pts.float()
+    diff = pts.unsqueeze(-2) - pts.unsqueeze(-3)
+    d2 = (diff * diff).sum(-1)
+    B, V, _ = d2.shape
+    cm = torch.zeros(B, V+1, V+1, device=d2.device, dtype=torch.float32)
+    cm[:, 0, 1:] = 1; cm[:, 1:, 0] = 1; cm[:, 1:, 1:] = d2
+    s = (-1.0)**V; f = math.factorial(V-1)
+    return s / ((2.0**(V-1)) * f*f) * torch.linalg.det(cm)
+
+def cv_loss(emb, target=0.12, n_samples=16):
+    B = emb.shape[0]
+    if B < 5: return torch.tensor(0.0, device=emb.device)
+    vols = []
+    for _ in range(n_samples):
+        idx = torch.randperm(B, device=emb.device)[:5]
+        v2 = cayley_menger_vol2(emb[idx].unsqueeze(0))
+        vols.append(torch.sqrt(F.relu(v2[0]) + 1e-12))
+    stacked = torch.stack(vols)
+    cv = stacked.std() / (stacked.mean() + 1e-8)
+    return (cv - target).abs()
+
+def cv_metric(emb, n=200):
+    B = emb.shape[0]
+    if B < 5: return 0.0
+    vols = []
+    for _ in range(n):
+        idx = torch.randperm(B, device=emb.device)[:5]
+        v2 = cayley_menger_vol2(emb[idx].unsqueeze(0))
+        v = torch.sqrt(F.relu(v2[0]) + 1e-12).item()
+        if v > 0: vols.append(v)
+    if len(vols) < 10: return 0.0
+    a = np.array(vols)
+    return float(a.std() / (a.mean() + 1e-8))
+
+def measure_consensus_stats(consensus_embs, n_check=2000):
+    """Measure exact geometric statistics of the consensus manifold."""
+    embs = consensus_embs[:n_check].float()
+    # CV
+    cv = cv_metric(embs.to(DEVICE))
+    # Pairwise cosine
+    sim = embs @ embs.T
+    mask = ~torch.eye(embs.shape[0], dtype=torch.bool)
+    pairwise = sim[mask]
+    mean_cos = pairwise.mean().item()
+    # Spectral
+    centered = embs - embs.mean(0, keepdim=True)
+    S = torch.linalg.svdvals(centered)
+    S_norm = (S / (S.sum() + 1e-8)).tolist()[:50]
+    # Eff dim
+    eff_dim = float((S.sum() ** 2) / (S.pow(2).sum() + 1e-12))
+
+    return {
+        "cv": cv,
+        "mean_cos": mean_cos,
+        "spectral": S_norm,
+        "eff_dim": eff_dim,
+    }
+
+
+# ══════════════════════════════════════════════════════════════════
+# EXTRACTION + ALIGNMENT
+# ══════════════════════════════════════════════════════════════════
+
+def symmetric_inv_sqrt(cov, eps=1e-6):
+    evals, evecs = torch.linalg.eigh(cov)
+    evals = torch.clamp(evals, min=eps)
+    return evecs @ torch.diag(evals.rsqrt()) @ evecs.T
+
+def procrustes_align(source, target, n_align=5000):
+    N = min(n_align, source.shape[0], target.shape[0])
+    S = source[:N].float(); T = target[:N].float()
+    s_mean = S.mean(0, keepdim=True); t_mean = T.mean(0, keepdim=True)
+    Sc = S - s_mean; Tc = T - t_mean; N_s = Sc.shape[0]
+    cos_before = F.cosine_similarity(Sc, Tc, dim=-1).mean().item()
+    s_cov = (Sc.T @ Sc) / max(N_s - 1, 1)
+    t_cov = (Tc.T @ Tc) / max(N_s - 1, 1)
+    s_whiten = symmetric_inv_sqrt(s_cov)
+    t_whiten = symmetric_inv_sqrt(t_cov)
+    Sc_w = F.normalize(Sc @ s_whiten, dim=-1)
+    Tc_w = F.normalize(Tc @ t_whiten, dim=-1)
+    U, _, Vt = torch.linalg.svd(Tc_w.T @ Sc_w, full_matrices=False)
+    R = U @ Vt
+    cos_after = F.cosine_similarity(Sc_w @ R.T, Tc_w, dim=-1).mean().item()
+    return {
+        "rotation": R, "source_mean": s_mean.squeeze(0),
+        "source_whitener": s_whiten,
+        "target_unwhitener": torch.linalg.pinv(t_whiten),
+        "cos_before": cos_before, "cos_after": cos_after,
+    }
+
+def apply_align(emb, a):
+    x = emb.float() - a["source_mean"]
+    x = x @ a["source_whitener"]; x = x @ a["rotation"].T
+    x = x @ a["target_unwhitener"]; return x
+
+
+# ══════════════════════════════════════════════════════════════════
+# MAIN
+# ══════════════════════════════════════════════════════════════════
+
+def run():
+    torch.manual_seed(42)
+    np.random.seed(42)
+    N_SAMPLES = 20000
+    MAX_LEN = 128
+    BATCH = 256
+
+    # ── Phase 0: Extract ──
+    print(f"\n{'='*65}")
+    print("PHASE 0: EXTRACTION")
+    print(f"{'='*65}")
+
+    from datasets import load_dataset
+    from transformers import AutoModel, AutoTokenizer
+
+    ds = load_dataset("CaptionEmporium/conceptual-captions-cc12m-llavanext",
+                      split="train", streaming=True)
+    captions = []
+    for row in ds:
+        cap = row.get("caption_llava", "")
+        if isinstance(cap, str) and len(cap) > 50:
+            captions.append(cap)
+        if len(captions) >= N_SAMPLES:
+            break
+    print(f"  Captions: {len(captions):,}")
+
+    embeds = {}
+    for model_name, short, max_len in EXPERTS:
+        print(f"\n  Extracting: {short}...")
+        model = AutoModel.from_pretrained(model_name).to(DEVICE).eval()
+        tokenizer = AutoTokenizer.from_pretrained(model_name)
+        all_emb = []
+        with torch.no_grad():
+            for i in tqdm(range(0, len(captions), 128), desc=f"    {short}"):
+                batch = captions[i:i+128]
+                inputs = tokenizer(batch, max_length=max_len, padding=True,
+                                  truncation=True, return_tensors="pt").to(DEVICE)
+                out = model(**inputs)
+                m = inputs.attention_mask.unsqueeze(-1).float()
+                pooled = (out.last_hidden_state * m).sum(1) / m.sum(1).clamp(min=1)
+                all_emb.append(pooled.cpu())
+        embeds[short] = torch.cat(all_emb)
+        print(f"    Shape: {embeds[short].shape}")
+        del model; gc.collect(); torch.cuda.empty_cache()
+
+    # ── Phase 0b: Align + Consensus + Measure ──
+    print(f"\n{'='*65}")
+    print("PHASE 0b: GENERALIZED PROCRUSTES ALIGNMENT (no reference bias)")
+    print(f"{'='*65}")
+
+    names = [s for _, s, _ in EXPERTS]
+
+    # Generalized Procrustes: iteratively align all to their mean
+    # No expert is the reference. The centerpoint emerges.
+    GPA_ITERS = 10
+    current = {name: embeds[name].float() for name in names}
+
+    for gpa_iter in range(GPA_ITERS):
+        # Compute mean shape
+        mean_shape = sum(current[n] for n in names) / len(names)
+
+        # Align each to mean
+        new_current = {}
+        total_delta = 0.0
+        for name in names:
+            info = procrustes_align(current[name], mean_shape)
+            new_current[name] = apply_align(current[name], info)
+            # Measure how much this iteration changed things
+            delta = (new_current[name] - current[name]).pow(2).mean().item()
+            total_delta += delta
+
+        current = new_current
+        if gpa_iter == 0 or (gpa_iter + 1) % 3 == 0 or total_delta < 1e-8:
+            print(f"  GPA iter {gpa_iter+1}: delta={total_delta:.8f}")
+        if total_delta < 1e-8:
+            print(f"  Converged at iteration {gpa_iter+1}")
+            break
+
+    # Final alignment: align each expert to the converged mean
+    mean_shape = sum(current[n] for n in names) / len(names)
+    procrustes_results = {}
+    aligned = {}
+    for name in names:
+        info = procrustes_align(embeds[name], mean_shape)
+        procrustes_results[name] = info
+        aligned[name] = apply_align(embeds[name], info)
+        cos = F.cosine_similarity(
+            aligned[name][:2000], mean_shape[:2000], dim=-1).mean().item()
+        print(f"  {name:10s}: cos_after={info['cos_after']:.4f}  cos_to_mean={cos:.4f}")
+
+    # Consensus = normalized centroid (now equidistant from all experts)
+    consensus = F.normalize(sum(aligned[n] for n in names) / len(names), dim=-1)
+    for name in names:
+        cos = F.cosine_similarity(consensus[:2000], aligned[name][:2000], dim=-1).mean().item()
+        print(f"  cos(consensus, {name}): {cos:.4f}")
+
+    # Verify equidistance
+    expert_cos_to_consensus = []
+    for name in names:
+        c = F.cosine_similarity(consensus[:2000], aligned[name][:2000], dim=-1).mean().item()
+        expert_cos_to_consensus.append(c)
+    equidist_range = max(expert_cos_to_consensus) - min(expert_cos_to_consensus)
+    print(f"  Equidistance range: {equidist_range:.4f} (should be near 0)")
+
+    # Measure EXACT consensus statistics
+    print(f"\n  Measuring consensus statistics...")
+    consensus_stats = measure_consensus_stats(consensus)
+    print(f"    CV:       {consensus_stats['cv']:.4f}")
+    print(f"    Mean cos: {consensus_stats['mean_cos']:.4f}")
+    print(f"    Eff dim:  {consensus_stats['eff_dim']:.1f}")
+    print(f"    Spectral: [{', '.join(f'{s:.4f}' for s in consensus_stats['spectral'][:5])}...]")
+
+    del embeds, aligned
+    gc.collect(); torch.cuda.empty_cache()
+
+    # ── Phase 1: Train Student ──
+    print(f"\n{'='*65}")
+    print("PHASE 1: TRAIN STUDENT")
+    print(f"{'='*65}")
+
+    tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased")
+    tokens = tokenizer(captions, max_length=MAX_LEN, padding="max_length",
+                       truncation=True, return_tensors="pt")
+    input_ids = tokens["input_ids"]
+    attention_mask = tokens["attention_mask"]
+
+    n_train = N_SAMPLES - 2000
+    train_ids = input_ids[:n_train].to(DEVICE)
+    train_mask = attention_mask[:n_train].to(DEVICE)
+    train_targets = consensus[:n_train].to(DEVICE)
+    val_ids = input_ids[n_train:].to(DEVICE)
+    val_mask = attention_mask[n_train:].to(DEVICE)
+    val_targets = consensus[n_train:].to(DEVICE)
+
+    student = MiniStudent(
+        vocab_size=tokenizer.vocab_size, max_len=MAX_LEN,
+        d_model=256, n_heads=4, n_layers=4, d_ff=1024,
+        output_dim=768, dropout=0.1, pad_token_id=tokenizer.pad_token_id
+    ).to(DEVICE)
+    n_params = sum(p.numel() for p in student.parameters())
+    print(f"  Student: {n_params:,} params")
+    print(f"  CV target: {consensus_stats['cv']:.4f}")
+
+    optimizer = torch.optim.AdamW(student.parameters(), lr=3e-4, weight_decay=0.01)
+
+    for epoch in range(5):
+        student.train()
+        perm = torch.randperm(n_train, device=DEVICE)
+        t_loss, t_acc, t_cos, n = 0, 0, 0, 0
+        t0 = time.time()
+        for i in range(0, n_train, BATCH):
+            idx = perm[i:i+BATCH]
+            if len(idx) < 8: continue
+            emb = student(train_ids[idx], train_mask[idx])
+            tgt = train_targets[idx]
+            l_nce, acc = infonce(emb, tgt)
+            l_mse = F.mse_loss(emb, tgt)
+            l_cv = cv_loss(emb, target=consensus_stats["cv"])
+            loss = l_nce + l_mse + 0.1 * l_cv
+            loss.backward()
+            torch.nn.utils.clip_grad_norm_(student.parameters(), 1.0)
+            optimizer.step(); optimizer.zero_grad(set_to_none=True)
+            with torch.no_grad():
+                cos = F.cosine_similarity(emb, tgt, dim=-1).mean().item()
+            t_loss += loss.item(); t_acc += acc; t_cos += cos; n += 1
+        elapsed = time.time() - t0; d = max(n, 1)
+        student.eval()
+        with torch.no_grad():
+            v_emb = student(val_ids, val_mask)
+            _, v_acc = infonce(v_emb[:1000], val_targets[:1000])
+            v_cos = F.cosine_similarity(v_emb, val_targets, dim=-1).mean().item()
+            v_cv = cv_metric(v_emb[:1000])
+        print(f"  E{epoch+1}: {elapsed:.0f}s  loss={t_loss/d:.4f}  "
+              f"t_acc={t_acc/d:.3f}  t_cos={t_cos/d:.3f}  "
+              f"v_acc={v_acc:.3f}  v_cos={v_cos:.3f}  v_cv={v_cv:.3f}")
+
+    torch.save(student.state_dict(), "mini_student.pt")
+    print(f"\n  Student saved. v_cos={v_cos:.3f}, v_cv={v_cv:.3f}")
+
+    # ── Phase 2: Train Alignment Bank ──
+    print(f"\n{'='*65}")
+    print("PHASE 2: TRAIN ALIGNMENT BANK (student frozen)")
+    print(f"{'='*65}")
+
+    student.eval()
+    for p in student.parameters():
+        p.requires_grad = False
+
+    print("  Pre-encoding through frozen student...")
+    with torch.no_grad():
+        all_embs = []
+        for i in range(0, n_train, 512):
+            j = min(i + 512, n_train)
+            emb = student(train_ids[i:j], train_mask[i:j])
+            all_embs.append(emb)
+        student_embs = torch.cat(all_embs)
+        val_student_embs = student(val_ids, val_mask)
+    print(f"  Student embeddings: {student_embs.shape}")
+
+    bank = AlignmentBank(
+        d_embed=768, n_experts=len(EXPERTS),
+        n_anchors=512, d_bank=128
+    ).to(DEVICE)
+
+    bank.init_from_procrustes(procrustes_results, names,
+                               consensus[:n_train], consensus_stats)
+    bank_params = sum(p.numel() for p in bank.parameters())
+    print(f"  Bank: {bank_params:,} params")
+    print(f"  Bank targets: CV={bank.target_cv.item():.4f}, "
+          f"mean_cos={bank.target_mean_cos.item():.4f}")
+
+    # Calibrate disagreement from initial state (before any training)
+    bank.calibrate_disagreement(student_embs[:2000])
+
+    bank_opt = torch.optim.AdamW(bank.parameters(), lr=1e-3, weight_decay=0.01)
+    BANK_EPOCHS = 20
+    BANK_BATCH = 256
+
+    for epoch in range(BANK_EPOCHS):
+        bank.train()
+        perm = torch.randperm(n_train, device=DEVICE)
+        total_loss = 0
+        stats = {"expert_agreement": 0, "rotation_ortho": 0,
+                 "anchor_spread": 0, "bank_cv": 0, "emb_cv": 0,
+                 "cross_expert_var": 0, "disagree_preserve": 0}
+        n = 0
+        t0 = time.time()
+        for i in range(0, n_train, BANK_BATCH):
+            idx = perm[i:i+BANK_BATCH]
+            if len(idx) < 16: continue
+            emb = student_embs[idx]
+            enriched, aux = bank(emb)
+            loss = bank.bank_loss(aux)
+            loss.backward()
+            torch.nn.utils.clip_grad_norm_(bank.parameters(), 1.0)
+            bank_opt.step(); bank_opt.zero_grad(set_to_none=True)
+            total_loss += loss.item()
+            for k in stats:
+                if k in aux:
+                    v = aux[k]
+                    stats[k] += v.item() if torch.is_tensor(v) else v
+            n += 1
+        elapsed = time.time() - t0; d = max(n, 1)
+
+        bank.eval()
+        with torch.no_grad():
+            v_enriched, v_aux = bank(val_student_embs)
+            v_loss = bank.bank_loss(v_aux).item()
+
+        print(f"\n  E{epoch+1:2d}: {elapsed:.0f}s  loss={total_loss/d:.4f}  v_loss={v_loss:.4f}")
+        print(f"    Geometry:  b_cv={stats['bank_cv']/d:.4f}  e_cv={stats['emb_cv']/d:.4f}  "
+              f"spread={stats['anchor_spread']/d:.5f}  a_max={v_aux['anchor_max_cos']:.3f}")
+        print(f"    Experts:   cos={v_aux['expert_cos_mean']:.3f}±{v_aux['expert_cos_std']:.3f}  "
+              f"agr={stats['expert_agreement']/d:.6f}  ortho={stats['rotation_ortho']/d:.6f}")
+        print(f"    Disagree:  x_cos={v_aux.get('cross_expert_cos', 0):.4f}±{v_aux.get('cross_expert_cos_std', 0):.4f}  "
+              f"ratio={v_aux['disagreement_ratio']:.6f}  "
+              f"preserve={stats['disagree_preserve']/d:.6f}  "
+              f"norms={v_aux['norm_ratio_spread']:.4f}")
+
+    torch.save(bank.state_dict(), "alignment_bank.pt")
+
+    # ── Phase 3: Geometric Verification ──
+    print(f"\n{'='*65}")
+    print("PHASE 3: GEOMETRIC VERIFICATION")
+    print(f"{'='*65}")
+
+    bank.eval()
+    with torch.no_grad():
+        enriched_val, v_aux = bank(val_student_embs)
+        original_768 = enriched_val[:, :768]
+        geo_context = enriched_val[:, 768:]
+
+        passthrough_cos = F.cosine_similarity(
+            original_768[:100], val_student_embs[:100], dim=-1).mean().item()
+        geo_cv = cv_metric(F.normalize(geo_context[:1000], dim=-1))
+        S = torch.linalg.svdvals(
+            geo_context[:1000].float() - geo_context[:1000].float().mean(0))
+        geo_eff_dim = float((S.sum() ** 2) / (S.pow(2).sum() + 1e-12))
+
+        # Verify consensus stats are preserved
+        emb_cv = cv_metric(val_student_embs[:1000])
+
+    print(f"  Passthrough:     {passthrough_cos:.6f} (target: 1.000)")
+    print(f"  Emb CV:          {emb_cv:.4f} (consensus: {consensus_stats['cv']:.4f})")
+    print(f"  Geo context CV:  {geo_cv:.4f}")
+    print(f"  Geo eff_dim:     {geo_eff_dim:.1f} / {bank.d_bank}")
+    print(f"  Expert cos:      {v_aux['expert_cos_mean']:.3f} ± {v_aux['expert_cos_std']:.3f}")
+    print(f"  Anchor max cos:  {v_aux['anchor_max_cos']:.3f}")
+    print(f"  Disagreement:")
+    print(f"    Cross-expert:  {v_aux.get('cross_expert_cos', 0):.4f} ± {v_aux.get('cross_expert_cos_std', 0):.4f}")
+    print(f"    Ratio:         {v_aux['disagreement_ratio']:.6f} (target: {bank.target_disagreement_ratio.item():.6f})")
+    print(f"    Norm spread:   {v_aux['norm_ratio_spread']:.4f}")
+
+    # ── Phase 4: Classifier Stability Test ──
+    print(f"\n{'='*65}")
+    print("PHASE 4: CLASSIFIER STABILITY TEST")
+    print(f"{'='*65}")
+
+    with torch.no_grad():
+        embs = val_student_embs[:1000]
+        sim = embs @ embs.T
+        sim.fill_diagonal_(-1)
+        n_pairs = 3000
+        idx_a = torch.randint(0, 1000, (n_pairs,))
+        idx_b = torch.randint(0, 1000, (n_pairs,))
+        pair_cos = sim[idx_a, idx_b]
+        sorted_cos, _ = pair_cos.sort()
+        t1 = sorted_cos[n_pairs // 3].item()
+        t2 = sorted_cos[2 * n_pairs // 3].item()
+        labels = torch.zeros(n_pairs, dtype=torch.long, device=DEVICE)
+        labels[pair_cos > t2] = 0
+        labels[(pair_cos <= t2) & (pair_cos > t1)] = 1
+        labels[pair_cos <= t1] = 2
+        enriched_a, _ = bank(embs[idx_a])
+        enriched_b, _ = bank(embs[idx_b])
+
+    for mode in ["with_bank", "without_bank"]:
+        if mode == "with_bank":
+            feat_dim = (768 + 128) * 2
+            features = torch.cat([enriched_a, enriched_b], dim=-1)
+        else:
+            feat_dim = 768 * 2
+            features = torch.cat([embs[idx_a], embs[idx_b]], dim=-1)
+
+        clf = nn.Sequential(
+            nn.Linear(feat_dim, 256), nn.GELU(), nn.LayerNorm(256),
+            nn.Linear(256, 3)
+        ).to(DEVICE)
+        clf_opt = torch.optim.Adam(clf.parameters(), lr=1e-3)
+        n_clf_train = 2400
+        train_f = features[:n_clf_train].detach()
+        train_l = labels[:n_clf_train]
+        val_f = features[n_clf_train:].detach()
+        val_l = labels[n_clf_train:]
+        for e in range(30):
+            clf.train()
+            logits = clf(train_f)
+            loss = F.cross_entropy(logits, train_l)
+            loss.backward(); clf_opt.step(); clf_opt.zero_grad()
+        clf.eval()
+        with torch.no_grad():
+            v_acc = (clf(val_f).argmax(-1) == val_l).float().mean().item()
+            t_acc = (clf(train_f).argmax(-1) == train_l).float().mean().item()
+        print(f"  {mode:15s}: train={t_acc:.3f}  val={v_acc:.3f}  gap={t_acc-v_acc:.3f}")
+
+    print(f"\n{'='*65}")
+    print("SUMMARY")
+    print(f"{'='*65}")
+    print(f"  Consensus CV:     {consensus_stats['cv']:.4f}")
+    print(f"  Consensus eff_dim:{consensus_stats['eff_dim']:.1f}")
+    print(f"  Student v_cos:    {v_cos:.3f}")
+    print(f"  Student v_cv:     {v_cv:.3f}")
+    print(f"  Bank params:      {bank_params:,}")
+    print(f"  Bank geo_eff_dim: {geo_eff_dim:.1f}")
+    print(f"  Bank geo_cv:      {geo_cv:.4f}")
+    print(f"\n{'='*65}")
+    print("DONE")
+    print(f"{'='*65}")
+
+
+if __name__ == "__main__":
+    run()