v1.2-traffic: router.json

router.json

@@ -1,15 +1,15 @@
 {
- "version": "v1.1-trained-live",
+ "version": "v1.2-traffic",
  "kind": "trained",
  "hf_repo": "experiential-labs/coding-router",
  "embed_model_mlx": "encoder-mlx-4bit",
  "embed_model_torch": "encoder-fp16",
- "n_tasks": 110,
+ "n_tasks": 207,
  "n_repos": 88,
  "embed_backend": "local",
  "T": 0.05453708011655199,
- "lam": 0.005,
- "sim_floor": 0.459,
+ "lam": 0.1,
+ "sim_floor": 0.4699,
  "fallback_arm_index": 5,
  "arms": [
   "fable5_xhigh",
@@ -144,48 +144,72 @@
    }
   }
  },
- "provenance": "EXP-012 winning recipe `reward_lcb_b0.2`: Qwen3-Embedding-0.6B with LoRA r=16 (q/k/v/o projections) trained by exact expected reward -E_pi[graded - 3.0*cost] on the LiveCodeBench 7x76 matrix, KL(pi||pi_init) anchor beta=0.2, 150 steps, checkpoint step 50 (T=0.0545), lam=0.01 -- (step, lam) selected ONLY on the six standard seeds' inner 75/25 DeepSWE-train splits (feasibility inner_graded >= inner-best-arm - 0.01, then max mean cost ratio; feasible on 5/6 inner splits). Memory/calibration side: the full 110-task DeepSWE v1.1 evidence (41 arms x 110 tasks, 88 repos), bank embedded with the merged fp16 encoder via sentence-transformers. Sweep evidence (per-seed selection, 6 seeds): selected-holdout mean graded 0.9484 at $60.17/split -- +0.015 graded and 2.1x cheaper than the deployable always-best-train baseline (0.9336/$126.28); parity, NOT better, vs the hindsight-best static arm. Cost figures are matrix-based (June 2026 collection) and pending live re-benchmark per EXP-014 (live drift measured). | v1.1 REBUILD (2026-08-01): bank cells replaced with the EXP-015 LIVE DeepSWE matrix (1,130 trials, dense 10 arms x 113 tasks; the 3 tasks without bank texts dropped -> 10x110) because the published June matrix is quality-stale live (e.g. luna_max 0.946 published vs 0.687 live). Roster 41 -> 10 live arms; med_cost/fallback from live cells (opus5_high, live f2p 0.951); tuned encoder, bank embeddings and T unchanged; lambda re-selected on inner train-side splits only (0.005).",
- "scope_warning": "INPUT SHAPE MATTERS. The decision rule was validated on repo-issue statements (p50 ~2,000 chars, long-horizon SWE tasks, median 61 agent steps). The encoder was tuned on LCB competitive-programming tasks and evaluated ONLY on the DeepSWE holdout; sim_floor abstains (escalates to the strongest arm) when no bank task is close in the TUNED space. Short one-line prompts will usually abstain -- safe, but no saving. Always check Decision.off_distribution.",
+ "provenance": "EXP-012 winning recipe `reward_lcb_b0.2`: Qwen3-Embedding-0.6B with LoRA r=16 (q/k/v/o projections) trained by exact expected reward -E_pi[graded - 3.0*cost] on the LiveCodeBench 7x76 matrix, KL(pi||pi_init) anchor beta=0.2, 150 steps, checkpoint step 50 (T=0.0545), lam=0.01 -- (step, lam) selected ONLY on the six standard seeds' inner 75/25 DeepSWE-train splits (feasibility inner_graded >= inner-best-arm - 0.01, then max mean cost ratio; feasible on 5/6 inner splits). Memory/calibration side: the full 110-task DeepSWE v1.1 evidence (41 arms x 110 tasks, 88 repos), bank embedded with the merged fp16 encoder via sentence-transformers. Sweep evidence (per-seed selection, 6 seeds): selected-holdout mean graded 0.9484 at $60.17/split -- +0.015 graded and 2.1x cheaper than the deployable always-best-train baseline (0.9336/$126.28); parity, NOT better, vs the hindsight-best static arm. Cost figures are matrix-based (June 2026 collection) and pending live re-benchmark per EXP-014 (live drift measured). | v1.1 REBUILD (2026-08-01): bank cells replaced with the EXP-015 LIVE DeepSWE matrix (1,130 trials, dense 10 arms x 113 tasks; the 3 tasks without bank texts dropped -> 10x110) because the published June matrix is quality-stale live (e.g. luna_max 0.946 published vs 0.687 live). Roster 41 -> 10 live arms; med_cost/fallback from live cells (opus5_high, live f2p 0.951); tuned encoder, bank embeddings and T unchanged; lambda re-selected on inner train-side splits only (0.005). | v1.2-traffic (2026-08-01): union bank = DeepSWE-live 110 (unchanged cells) + 65 LCB-live + 32 utility tasks, all arms measured LIVE this lane (LCB via the E2B harness on the 92-task covered subset; utility = 32 short interactive asks with deterministic verifiers on 4 pinned OSS repos). graded = harness fraction; med_cost per arm = median of live per-task costs pooled across the three sources. sim_floor = LOO-nearest p10 over the union bank. lam re-selected train-side only under a 50/30/20 utility/LCB/DeepSWE traffic weighting.",
+ "scope_warning": "Validated on repo-issue statements (DeepSWE), competitive-programming agent tasks (LCB) and SHORT interactive asks (utility suite: bugfix/feature/refactor/question, 40-250 chars) \u2014 short prompts now land in-distribution instead of abstaining. Abstention (escalate to strongest arm) is reserved for text unlike any bank task; always check Decision.off_distribution.",
  "selection": {
   "lam_grid": [
    {
     "lam": 0.001,
-    "feas": 5,
-    "mean_ratio": 1.3443251848220825,
-    "mean_inner_graded": 0.9251648088296255
+    "feas": 6,
+    "mean_ratio": 1.081393772849211,
+    "mean_inner_graded": 0.7660379605122892
    },
    {
     "lam": 0.002,
-    "feas": 5,
-    "mean_ratio": 1.4847139120101929,
-    "mean_inner_graded": 0.9296404520670573
+    "feas": 6,
+    "mean_ratio": 1.1000667545641503,
+    "mean_inner_graded": 0.7659905447036618
    },
    {
     "lam": 0.005,
     "feas": 6,
-    "mean_ratio": 2.115150213241577,
-    "mean_inner_graded": 0.9336468180020651
+    "mean_ratio": 1.2811509404979988,
+    "mean_inner_graded": 0.765745203615449
    },
    {
     "lam": 0.01,
-    "feas": 5,
-    "mean_ratio": 4.753244400024414,
-    "mean_inner_graded": 0.9279556373755137
+    "feas": 6,
+    "mean_ratio": 1.7326780876087458,
+    "mean_inner_graded": 0.7656267731370289
    },
    {
     "lam": 0.02,
-    "feas": 4,
-    "mean_ratio": 19.882097244262695,
-    "mean_inner_graded": 0.9321131110191345
+    "feas": 6,
+    "mean_ratio": 2.452558818623846,
+    "mean_inner_graded": 0.7645313997014641
    },
    {
     "lam": 0.05,
-    "feas": 4,
-    "mean_ratio": 31.052934646606445,
-    "mean_inner_graded": 0.9164654314517975
+    "feas": 6,
+    "mean_ratio": 3.2512255254226896,
+    "mean_inner_graded": 0.7644669476479534
+   },
+   {
+    "lam": 0.1,
+    "feas": 6,
+    "mean_ratio": 3.5954684774287116,
+    "mean_inner_graded": 0.7617608602098715
+   },
+   {
+    "lam": 0.2,
+    "feas": 5,
+    "mean_ratio": 3.9614547647054508,
+    "mean_inner_graded": 0.7608279933293333
+   },
+   {
+    "lam": 0.5,
+    "feas": 5,
+    "mean_ratio": 4.002295380963825,
+    "mean_inner_graded": 0.7593800766626666
+   },
+   {
+    "lam": 1.0,
+    "feas": 5,
+    "mean_ratio": 4.061556689358738,
+    "mean_inner_graded": 0.7589185461017406
    }
   ],
-  "selected_lam": 0.005,
-  "policy": "same inner-split protocol as the mint; encoder/T frozen"
+  "selected_lam": 0.1,
+  "policy": "6 seeded group-stratified splits, inner train-side only; metric weights sources 50% utility / 30% LCB / 20% DeepSWE (assumed traffic mix, documented); encoder/T frozen"
  }
 }