{
  "checkpoint": "/workspace/coding_decision_agent/run",
  "n_cases": 14344,
  "max_len": 2048,
  "head_max_len": 320,
  "models": {
    "coding_decision_agent": {
      "ALL": {
        "n": 41557,
        "accuracy": 0.7810236542580071,
        "soft_accuracy": 0.6169353151939833,
        "brier": 0.14093584434109901,
        "ece": 0.05023568592535557,
        "score_mae": 0.28701420688035695,
        "latency_ms_p50": 12.195211059103409
      },
      "agent_trace": {
        "n": 500,
        "accuracy": 0.738,
        "soft_accuracy": 0.4474543152773224,
        "brier": 0.045567078348232405,
        "ece": 0.17601319999999998,
        "score_mae": 0.295,
        "latency_ms_p50": 5.2139530424028635
      },
      "agent_trace.action": {
        "n": 100,
        "accuracy": 0.71,
        "soft_accuracy": 0.3801464152595509,
        "brier": 0.04073350452343815,
        "ece": 0.20638399999999996,
        "score_mae": null,
        "latency_ms_p50": 5.2139530424028635
      },
      "agent_trace.needs_review": {
        "n": 100,
        "accuracy": 0.87,
        "soft_accuracy": 0.625292183322,
        "brier": 0.029521532578379993,
        "ece": 0.18256499999999992,
        "score_mae": null,
        "latency_ms_p50": 5.2139530424028635
      },
      "agent_trace.outcome": {
        "n": 100,
        "accuracy": 0.7,
        "soft_accuracy": 0.407664397596501,
        "brier": 0.06431531813416594,
        "ece": 0.251432,
        "score_mae": null,
        "latency_ms_p50": 5.2139530424028635
      },
      "agent_trace.risk": {
        "n": 100,
        "accuracy": 0.76,
        "soft_accuracy": 0.4461210768917738,
        "brier": 0.04723240428914765,
        "ece": 0.18484399999999995,
        "score_mae": 0.24,
        "latency_ms_p50": 5.2139530424028635
      },
      "agent_trace.urgency": {
        "n": 100,
        "accuracy": 0.65,
        "soft_accuracy": 0.3780475033167836,
        "brier": 0.046032632216030205,
        "ece": 0.15192499999999998,
        "score_mae": 0.35,
        "latency_ms_p50": 5.2139530424028635
      },
      "code_review": {
        "n": 13905,
        "accuracy": 0.6005753326141675,
        "soft_accuracy": 0.43762436286692236,
        "brier": 0.20462145956578426,
        "ece": 0.031638755843221865,
        "score_mae": 0.8204962243797195,
        "latency_ms_p50": 11.18671971683701
      },
      "code_review.code_quality": {
        "n": 4635,
        "accuracy": 0.45674217907227616,
        "soft_accuracy": 0.3393588623455584,
        "brier": 0.31392093132792775,
        "ece": 0.0472170442286947,
        "score_mae": 0.8204962243797195,
        "latency_ms_p50": 11.18671971683701
      },
      "code_review.instruction_followed": {
        "n": 2333,
        "accuracy": 0.7513930561508787,
        "soft_accuracy": 0.5743438717384003,
        "brier": 0.11188956051875844,
        "ece": 0.06876155165023576,
        "score_mae": null,
        "latency_ms_p50": 11.241163748006025
      },
      "code_review.likely_correct": {
        "n": 2302,
        "accuracy": 0.7528236316246742,
        "soft_accuracy": 0.6613456443825189,
        "brier": 0.27179493373129193,
        "ece": 0.079262858384014,
        "score_mae": null,
        "latency_ms_p50": 11.151967492575448
      },
      "code_review.merge_action": {
        "n": 4635,
        "accuracy": 0.5928802588996763,
        "soft_accuracy": 0.35596041263400946,
        "brier": 0.10863594313216428,
        "ece": 0.10759497303128371,
        "score_mae": null,
        "latency_ms_p50": 11.18671971683701
      },
      "routing": {
        "n": 2121,
        "accuracy": 0.6454502593116455,
        "soft_accuracy": 0.4342094103303752,
        "brier": 0.16347383356431308,
        "ece": 0.10083290900518624,
        "score_mae": 0.314002828854314,
        "latency_ms_p50": 8.60158633440733
      },
      "routing.model_tier": {
        "n": 707,
        "accuracy": 0.4186704384724187,
        "soft_accuracy": 0.29104883312034857,
        "brier": 0.2916796882664165,
        "ece": 0.03434865629420089,
        "score_mae": null,
        "latency_ms_p50": 8.60158633440733
      },
      "routing.skill": {
        "n": 707,
        "accuracy": 0.8118811881188119,
        "soft_accuracy": 0.6114870834079437,
        "brier": 0.09689897214124599,
        "ece": 0.11263097595473832,
        "score_mae": null,
        "latency_ms_p50": 8.60158633440733
      },
      "routing.task_difficulty": {
        "n": 707,
        "accuracy": 0.7057991513437057,
        "soft_accuracy": 0.40009231446283555,
        "brier": 0.1018428402852769,
        "ece": 0.1647117397454031,
        "score_mae": 0.314002828854314,
        "latency_ms_p50": 8.60158633440733
      },
      "tool_call": {
        "n": 25031,
        "accuracy": 0.8936119212176901,
        "soft_accuracy": 0.7354132400184741,
        "brier": 0.10555303215440064,
        "ece": 0.058768491071071884,
        "score_mae": 0.07004872955099199,
        "latency_ms_p50": 12.551800115033984
      },
      "tool_call.call_verdict": {
        "n": 7033,
        "accuracy": 0.8312242286364283,
        "soft_accuracy": 0.6229654360153449,
        "brier": 0.132674780562995,
        "ece": 0.08592636143893075,
        "score_mae": null,
        "latency_ms_p50": 12.655832339078188
      },
      "tool_call.parameter_structure": {
        "n": 2873,
        "accuracy": 0.9425687434737209,
        "soft_accuracy": 0.8093260223974982,
        "brier": 0.0475529054278188,
        "ece": 0.06314893839192472,
        "score_mae": 0.05986773407587887,
        "latency_ms_p50": 11.679318360984325
      },
      "tool_call.query_coverage": {
        "n": 2873,
        "accuracy": 0.9644970414201184,
        "soft_accuracy": 0.8103391947839732,
        "brier": 0.04152526163506391,
        "ece": 0.07684267316394014,
        "score_mae": 0.03550295857988166,
        "latency_ms_p50": 11.679318360984325
      },
      "tool_call.sequence_accuracy": {
        "n": 2873,
        "accuracy": 0.907761921336582,
        "soft_accuracy": 0.7895875597673252,
        "brier": 0.11586305134957572,
        "ece": 0.053443264879916456,
        "score_mae": 0.10442046641141664,
        "latency_ms_p50": 11.679318360984325
      },
      "tool_call.should_call_tool": {
        "n": 6506,
        "accuracy": 0.8882569935444206,
        "soft_accuracy": 0.7544548504377908,
        "brier": 0.14932512450159993,
        "ece": 0.05518422994159238,
        "score_mae": null,
        "latency_ms_p50": 13.125395790363353
      },
      "tool_call.tool_selection": {
        "n": 2873,
        "accuracy": 0.9244691959624086,
        "soft_accuracy": 0.7645480035796159,
        "brier": 0.05175455469921832,
        "ece": 0.08405972850678739,
        "score_mae": 0.0804037591367908,
        "latency_ms_p50": 11.679318360984325
      }
    },
    "majority": {
      "ALL": {
        "n": 41557,
        "accuracy": 0.5345669802921289,
        "soft_accuracy": 0.49801396628013467,
        "brier": 0.7437008553795154,
        "ece": 0.4654330197078711,
        "score_mae": 0.5466126570388634,
        "latency_ms_p50": null
      },
      "agent_trace": {
        "n": 500,
        "accuracy": 0.36,
        "soft_accuracy": 0.36320601275334635,
        "brier": 0.7617543466908759,
        "ece": 0.64,
        "score_mae": 0.845,
        "latency_ms_p50": null
      },
      "agent_trace.action": {
        "n": 100,
        "accuracy": 0.34,
        "soft_accuracy": 0.3136333298333426,
        "brier": 0.7869731316669766,
        "ece": 0.6599999999999999,
        "score_mae": null,
        "latency_ms_p50": null
      },
      "agent_trace.needs_review": {
        "n": 100,
        "accuracy": 0.55,
        "soft_accuracy": 0.5427667100000001,
        "brier": 0.5737183794223801,
        "ece": 0.44999999999999996,
        "score_mae": null,
        "latency_ms_p50": null
      },
      "agent_trace.outcome": {
        "n": 100,
        "accuracy": 0.26,
        "soft_accuracy": 0.3122366658666861,
        "brier": 0.8482844572016827,
        "ece": 0.74,
        "score_mae": null,
        "latency_ms_p50": null
      },
      "agent_trace.risk": {
        "n": 100,
        "accuracy": 0.3,
        "soft_accuracy": 0.30959668803336143,
        "brier": 0.861741757986497,
        "ece": 0.7,
        "score_mae": 0.88,
        "latency_ms_p50": null
      },
      "agent_trace.urgency": {
        "n": 100,
        "accuracy": 0.35,
        "soft_accuracy": 0.3377966700333401,
        "brier": 0.7380540071768454,
        "ece": 0.65,
        "score_mae": 0.81,
        "latency_ms_p50": null
      },
      "code_review": {
        "n": 13905,
        "accuracy": 0.48507731032002877,
        "soft_accuracy": 0.42916732792419915,
        "brier": 0.75125638155265,
        "ece": 0.5149226896799712,
        "score_mae": 1.0017259978425026,
        "latency_ms_p50": null
      },
      "code_review.code_quality": {
        "n": 4635,
        "accuracy": 0.3326860841423948,
        "soft_accuracy": 0.29555703021936686,
        "brier": 1.0224402787502134,
        "ece": 0.6673139158576051,
        "score_mae": 1.0017259978425026,
        "latency_ms_p50": null
      },
      "code_review.instruction_followed": {
        "n": 2333,
        "accuracy": 0.5662237462494643,
        "soft_accuracy": 0.5045862671206824,
        "brier": 0.6618268633733213,
        "ece": 0.43377625375053575,
        "score_mae": null,
        "latency_ms_p50": null
      },
      "code_review.likely_correct": {
        "n": 2302,
        "accuracy": 0.7532580364900087,
        "soft_accuracy": 0.7127367506516157,
        "brier": 0.4273264986967823,
        "ece": 0.24674196350999134,
        "score_mae": null,
        "latency_ms_p50": null
      },
      "code_review.merge_action": {
        "n": 4635,
        "accuracy": 0.4634304207119741,
        "soft_accuracy": 0.3839795250326813,
        "brier": 0.6859679873208044,
        "ece": 0.5365695792880258,
        "score_mae": null,
        "latency_ms_p50": null
      },
      "routing": {
        "n": 2121,
        "accuracy": 0.4526166902404526,
        "soft_accuracy": 0.3796902013726896,
        "brier": 0.8319999746512764,
        "ece": 0.5473833097595474,
        "score_mae": 0.6760961810466761,
        "latency_ms_p50": null
      },
      "routing.model_tier": {
        "n": 707,
        "accuracy": 0.4200848656294201,
        "soft_accuracy": 0.3065111727338155,
        "brier": 0.9638235895227224,
        "ece": 0.57991513437058,
        "score_mae": null,
        "latency_ms_p50": null
      },
      "routing.skill": {
        "n": 707,
        "accuracy": 0.5487977369165488,
        "soft_accuracy": 0.48265260642916186,
        "brier": 0.737355320469855,
        "ece": 0.45120226308345124,
        "score_mae": null,
        "latency_ms_p50": null
      },
      "routing.task_difficulty": {
        "n": 707,
        "accuracy": 0.38896746817538896,
        "soft_accuracy": 0.34990682495509523,
        "brier": 0.7948210139612376,
        "ece": 0.611032531824611,
        "score_mae": 0.6760961810466761,
        "latency_ms_p50": null
      },
      "tool_call": {
        "n": 25031,
        "accuracy": 0.5724901122607966,
        "soft_accuracy": 0.5489780183944137,
        "brier": 0.7316610339955163,
        "ece": 0.4275098877392034,
        "score_mae": 0.3498955795335886,
        "latency_ms_p50": null
      },
      "tool_call.call_verdict": {
        "n": 7033,
        "accuracy": 0.5384615384615384,
        "soft_accuracy": 0.4594399893495609,
        "brier": 0.8238847454614278,
        "ece": 0.46153846153846156,
        "score_mae": null,
        "latency_ms_p50": null
      },
      "tool_call.parameter_structure": {
        "n": 2873,
        "accuracy": 0.8517229376957883,
        "soft_accuracy": 0.8523165100359604,
        "brier": 0.15079659078779242,
        "ece": 0.14827706230421167,
        "score_mae": 0.16498433693003828,
        "latency_ms_p50": null
      },
      "tool_call.query_coverage": {
        "n": 2873,
        "accuracy": 0.7514792899408284,
        "soft_accuracy": 0.7276519317786235,
        "brier": 0.39613735576439596,
        "ece": 0.24852071005917165,
        "score_mae": 0.2770623042116255,
        "latency_ms_p50": null
      },
      "tool_call.sequence_accuracy": {
        "n": 2873,
        "accuracy": 0.6244343891402715,
        "soft_accuracy": 0.5982352941176617,
        "brier": 0.6889294117646958,
        "ece": 0.3755656108597285,
        "score_mae": 0.5078315349808562,
        "latency_ms_p50": null
      },
      "tool_call.should_call_tool": {
        "n": 6506,
        "accuracy": 0.3556716876729173,
        "soft_accuracy": 0.3472179526590886,
        "brier": 1.1903640946817313,
        "ece": 0.6443283123270827,
        "score_mae": null,
        "latency_ms_p50": null
      },
      "tool_call.tool_selection": {
        "n": 2873,
        "accuracy": 0.6366167768882701,
        "soft_accuracy": 0.6937862861120654,
        "brier": 0.4262727806009997,
        "ece": 0.3633832231117299,
        "score_mae": 0.44970414201183434,
        "latency_ms_p50": null
      }
    },
    "zero_shot": {
      "ALL": {
        "n": 41557,
        "accuracy": 0.4495512188079024,
        "soft_accuracy": 0.3555622636666341,
        "brier": 0.4759449766209747,
        "ece": 0.13287126837837185,
        "score_mae": 0.6015028765997417,
        "latency_ms_p50": 12.262549810111523
      },
      "agent_trace": {
        "n": 500,
        "accuracy": 0.384,
        "soft_accuracy": 0.3383222443417828,
        "brier": 0.23905442880209826,
        "ece": 0.1748602,
        "score_mae": 0.855,
        "latency_ms_p50": 5.243273265659809
      },
      "agent_trace.action": {
        "n": 100,
        "accuracy": 0.34,
        "soft_accuracy": 0.2753353724584661,
        "brier": 0.22138194844624695,
        "ece": 0.18946300000000002,
        "score_mae": null,
        "latency_ms_p50": 5.243273265659809
      },
      "agent_trace.needs_review": {
        "n": 100,
        "accuracy": 0.47,
        "soft_accuracy": 0.5049384545179999,
        "brier": 0.17746708898638006,
        "ece": 0.14444700000000002,
        "score_mae": null,
        "latency_ms_p50": 5.243273265659809
      },
      "agent_trace.outcome": {
        "n": 100,
        "accuracy": 0.45,
        "soft_accuracy": 0.31141803381489647,
        "brier": 0.18086825047871102,
        "ece": 0.14976499999999995,
        "score_mae": null,
        "latency_ms_p50": 5.243273265659809
      },
      "agent_trace.risk": {
        "n": 100,
        "accuracy": 0.31,
        "soft_accuracy": 0.2871761327055934,
        "brier": 0.25651174481546657,
        "ece": 0.177668,
        "score_mae": 0.9,
        "latency_ms_p50": 5.243273265659809
      },
      "agent_trace.urgency": {
        "n": 100,
        "accuracy": 0.35,
        "soft_accuracy": 0.3127432282119582,
        "brier": 0.3590431112836864,
        "ece": 0.38155000000000006,
        "score_mae": 0.81,
        "latency_ms_p50": 5.243273265659809
      },
      "code_review": {
        "n": 13905,
        "accuracy": 0.2746494066882416,
        "soft_accuracy": 0.3079304536603386,
        "brier": 0.3907150229664912,
        "ece": 0.2118814383315354,
        "score_mae": 1.2159654800431499,
        "latency_ms_p50": 11.264183248082796
      },
      "code_review.code_quality": {
        "n": 4635,
        "accuracy": 0.2614886731391586,
        "soft_accuracy": 0.22136276379835138,
        "brier": 0.4098319305398359,
        "ece": 0.039814412081984885,
        "score_mae": 1.2159654800431499,
        "latency_ms_p50": 11.264183248082796
      },
      "code_review.instruction_followed": {
        "n": 2333,
        "accuracy": 0.45906558079725673,
        "soft_accuracy": 0.5059872505305001,
        "brier": 0.3192942502356396,
        "ece": 0.29432760394342045,
        "score_mae": null,
        "latency_ms_p50": 11.303223669528961
      },
      "code_review.likely_correct": {
        "n": 2302,
        "accuracy": 0.35621198957428324,
        "soft_accuracy": 0.43913897132927865,
        "brier": 0.6004540551520414,
        "ece": 0.36725212858384015,
        "score_mae": null,
        "latency_ms_p50": 11.229163501411676
      },
      "code_review.merge_action": {
        "n": 4635,
        "accuracy": 0.15447680690399138,
        "soft_accuracy": 0.22964193753052564,
        "brier": 0.30337921802313095,
        "ece": 0.2667025458468177,
        "score_mae": null,
        "latency_ms_p50": 11.264183248082796
      },
      "routing": {
        "n": 2121,
        "accuracy": 0.36020744931636023,
        "soft_accuracy": 0.272360666871322,
        "brier": 0.3998127569826348,
        "ece": 0.113611315417256,
        "score_mae": 0.545968882602546,
        "latency_ms_p50": 8.667557345082363
      },
      "routing.model_tier": {
        "n": 707,
        "accuracy": 0.3521923620933522,
        "soft_accuracy": 0.2723404817763805,
        "brier": 0.4081496013531005,
        "ece": 0.15772984441301272,
        "score_mae": null,
        "latency_ms_p50": 8.667557345082363
      },
      "routing.skill": {
        "n": 707,
        "accuracy": 0.22065063649222066,
        "soft_accuracy": 0.20394731084366793,
        "brier": 0.6251532200156299,
        "ece": 0.23935954738330978,
        "score_mae": null,
        "latency_ms_p50": 8.667557345082363
      },
      "routing.task_difficulty": {
        "n": 707,
        "accuracy": 0.5077793493635078,
        "soft_accuracy": 0.340794207993919,
        "brier": 0.16613544957917267,
        "ece": 0.06291711456859972,
        "score_mae": 0.545968882602546,
        "latency_ms_p50": 8.667557345082363
      },
      "tool_call": {
        "n": 25031,
        "accuracy": 0.5555910670768247,
        "soft_accuracy": 0.3894167207239965,
        "brier": 0.5344741691153987,
        "ece": 0.20038942910790614,
        "score_mae": 0.35268012530455967,
        "latency_ms_p50": 12.63355859555304
      },
      "tool_call.call_verdict": {
        "n": 7033,
        "accuracy": 0.5090288639272004,
        "soft_accuracy": 0.2763339934647179,
        "brier": 0.4873584676179949,
        "ece": 0.14773786435376085,
        "score_mae": null,
        "latency_ms_p50": 12.750410009175539
      },
      "tool_call.parameter_structure": {
        "n": 2873,
        "accuracy": 0.84963452836756,
        "soft_accuracy": 0.5603033966443252,
        "brier": 0.2091076900711842,
        "ece": 0.23563003828750437,
        "score_mae": 0.16916115558649494,
        "latency_ms_p50": 11.710590030997992
      },
      "tool_call.query_coverage": {
        "n": 2873,
        "accuracy": 0.7434737208492864,
        "soft_accuracy": 0.5031096529296475,
        "brier": 0.3056886843956699,
        "ece": 0.15529136790810996,
        "score_mae": 0.28402366863905326,
        "latency_ms_p50": 11.710590030997992
      },
      "tool_call.sequence_accuracy": {
        "n": 2873,
        "accuracy": 0.6108597285067874,
        "soft_accuracy": 0.42309666497926895,
        "brier": 0.4585507036080716,
        "ece": 0.08191855203619908,
        "score_mae": 0.5053950574312566,
        "latency_ms_p50": 11.710590030997992
      },
      "tool_call.should_call_tool": {
        "n": 6506,
        "accuracy": 0.3336919766369505,
        "soft_accuracy": 0.3170915542829213,
        "brier": 0.9759323309997293,
        "ece": 0.502032047340916,
        "score_mae": null,
        "latency_ms_p50": 13.202114030718803
      },
      "tool_call.tool_selection": {
        "n": 2873,
        "accuracy": 0.6348764357814132,
        "soft_accuracy": 0.5117622340163136,
        "brier": 0.2801910628530002,
        "ece": 0.01890006961364427,
        "score_mae": 0.45214061956143403,
        "latency_ms_p50": 11.710590030997992
      }
    },
    "typed": {
      "ALL": {
        "n": 41557,
        "accuracy": 0.45051375219577927,
        "soft_accuracy": 0.3390969306632602,
        "brier": 0.40869457266376125,
        "ece": 0.055596229275452984,
        "score_mae": 0.6435364564987671,
        "latency_ms_p50": 12.301214970648289
      },
      "agent_trace": {
        "n": 500,
        "accuracy": 0.714,
        "soft_accuracy": 0.4185213375514336,
        "brier": 0.05229190380897771,
        "ece": 0.2114094,
        "score_mae": 0.315,
        "latency_ms_p50": 5.285132722929121
      },
      "agent_trace.action": {
        "n": 100,
        "accuracy": 0.66,
        "soft_accuracy": 0.3378858933708014,
        "brier": 0.056391282386058256,
        "ece": 0.25632999999999995,
        "score_mae": null,
        "latency_ms_p50": 5.285132722929121
      },
      "agent_trace.needs_review": {
        "n": 100,
        "accuracy": 0.88,
        "soft_accuracy": 0.6068831542299998,
        "brier": 0.033795292162379996,
        "ece": 0.20902199999999999,
        "score_mae": null,
        "latency_ms_p50": 5.285132722929121
      },
      "agent_trace.outcome": {
        "n": 100,
        "accuracy": 0.66,
        "soft_accuracy": 0.3640242031319801,
        "brier": 0.07764432812338702,
        "ece": 0.221677,
        "score_mae": null,
        "latency_ms_p50": 5.285132722929121
      },
      "agent_trace.risk": {
        "n": 100,
        "accuracy": 0.75,
        "soft_accuracy": 0.41817244025729494,
        "brier": 0.04732115045857487,
        "ece": 0.223152,
        "score_mae": 0.25,
        "latency_ms_p50": 5.285132722929121
      },
      "agent_trace.urgency": {
        "n": 100,
        "accuracy": 0.62,
        "soft_accuracy": 0.36564099676709183,
        "brier": 0.046307465914488456,
        "ece": 0.14686600000000005,
        "score_mae": 0.38,
        "latency_ms_p50": 5.285132722929121
      },
      "code_review": {
        "n": 13905,
        "accuracy": 0.2575332614167566,
        "soft_accuracy": 0.3145546199361667,
        "brier": 0.33256056458936434,
        "ece": 0.1680778784609853,
        "score_mae": 1.3598705501618122,
        "latency_ms_p50": 11.288903653621674
      },
      "code_review.code_quality": {
        "n": 4635,
        "accuracy": 0.2161812297734628,
        "soft_accuracy": 0.2156468273418538,
        "brier": 0.4133553874242992,
        "ece": 0.0773525134843581,
        "score_mae": 1.3598705501618122,
        "latency_ms_p50": 11.288903653621674
      },
      "code_review.instruction_followed": {
        "n": 2333,
        "accuracy": 0.45049292756108017,
        "soft_accuracy": 0.5031287105466815,
        "brier": 0.22465357059209978,
        "ece": 0.20252091727389623,
        "score_mae": null,
        "latency_ms_p50": 11.332182368884483
      },
      "code_review.likely_correct": {
        "n": 2302,
        "accuracy": 0.32493483927019984,
        "soft_accuracy": 0.4597893605560382,
        "brier": 0.4740768352997398,
        "ece": 0.2952202867072111,
        "score_mae": null,
        "latency_ms_p50": 11.257084629808864
      },
      "code_review.merge_action": {
        "n": 4635,
        "accuracy": 0.16828478964401294,
        "soft_accuracy": 0.2464130648926627,
        "brier": 0.2357952049303329,
        "ece": 0.1784133764832794,
        "score_mae": null,
        "latency_ms_p50": 11.288903653621674
      },
      "routing": {
        "n": 2121,
        "accuracy": 0.35454974068835454,
        "soft_accuracy": 0.26241057114053784,
        "brier": 0.3317003871036206,
        "ece": 0.026113531353135305,
        "score_mae": 0.5318246110325319,
        "latency_ms_p50": 8.731171023100615
      },
      "routing.model_tier": {
        "n": 707,
        "accuracy": 0.272984441301273,
        "soft_accuracy": 0.2581747552619435,
        "brier": 0.3383839440346083,
        "ece": 0.08653380480905236,
        "score_mae": null,
        "latency_ms_p50": 8.731171023100615
      },
      "routing.skill": {
        "n": 707,
        "accuracy": 0.2871287128712871,
        "soft_accuracy": 0.20951270931679064,
        "brier": 0.4897188027773141,
        "ece": 0.0719220650636492,
        "score_mae": null,
        "latency_ms_p50": 8.731171023100615
      },
      "routing.task_difficulty": {
        "n": 707,
        "accuracy": 0.5035360678925035,
        "soft_accuracy": 0.3195442488428794,
        "brier": 0.1669984144989407,
        "ece": 0.08971980198019801,
        "score_mae": 0.5318246110325319,
        "latency_ms_p50": 8.731171023100615
      },
      "tool_call": {
        "n": 25031,
        "accuracy": 0.5605848747553034,
        "soft_accuracy": 0.35764195066900195,
        "brier": 0.464631186633436,
        "ece": 0.09784442091806163,
        "score_mae": 0.3672119735468152,
        "latency_ms_p50": 12.692837044596672
      },
      "tool_call.call_verdict": {
        "n": 7033,
        "accuracy": 0.5292193942840893,
        "soft_accuracy": 0.27012174415225865,
        "brier": 0.4859364810128098,
        "ece": 0.15381443196360017,
        "score_mae": null,
        "latency_ms_p50": 12.818309012800455
      },
      "tool_call.parameter_structure": {
        "n": 2873,
        "accuracy": 0.8419770274973895,
        "soft_accuracy": 0.495495654291868,
        "brier": 0.27895918554296956,
        "ece": 0.31188656456665514,
        "score_mae": 0.17577445179255133,
        "latency_ms_p50": 11.742415837943554
      },
      "tool_call.query_coverage": {
        "n": 2873,
        "accuracy": 0.7076226940480335,
        "soft_accuracy": 0.4637026902444456,
        "brier": 0.3405009573564468,
        "ece": 0.17881566306996174,
        "score_mae": 0.3198746954403063,
        "latency_ms_p50": 11.742415837943554
      },
      "tool_call.sequence_accuracy": {
        "n": 2873,
        "accuracy": 0.5767490428123913,
        "soft_accuracy": 0.39998245191225323,
        "brier": 0.47585696271728206,
        "ece": 0.10643915767490429,
        "score_mae": 0.5248868778280543,
        "latency_ms_p50": 11.742415837943554
      },
      "tool_call.should_call_tool": {
        "n": 6506,
        "accuracy": 0.3647402397786658,
        "soft_accuracy": 0.2743444411062642,
        "brier": 0.6462853975060024,
        "ece": 0.07603150937596063,
        "score_mae": null,
        "latency_ms_p50": 13.254288118332624
      },
      "tool_call.tool_selection": {
        "n": 2873,
        "accuracy": 0.6362687086668987,
        "soft_accuracy": 0.4742631669122307,
        "brier": 0.2996913056204823,
        "ece": 0.07137298990602153,
        "score_mae": 0.44831186912634874,
        "latency_ms_p50": 11.742415837943554
      }
    }
  }
}