| { |
| "checkpoint": "/workspace/coding_decision_agent/run", |
| "n_cases": 14344, |
| "max_len": 2048, |
| "head_max_len": 320, |
| "models": { |
| "coding_decision_agent": { |
| "ALL": { |
| "n": 41557, |
| "accuracy": 0.7810236542580071, |
| "soft_accuracy": 0.6169353151939833, |
| "brier": 0.14093584434109901, |
| "ece": 0.05023568592535557, |
| "score_mae": 0.28701420688035695, |
| "latency_ms_p50": 12.195211059103409 |
| }, |
| "agent_trace": { |
| "n": 500, |
| "accuracy": 0.738, |
| "soft_accuracy": 0.4474543152773224, |
| "brier": 0.045567078348232405, |
| "ece": 0.17601319999999998, |
| "score_mae": 0.295, |
| "latency_ms_p50": 5.2139530424028635 |
| }, |
| "agent_trace.action": { |
| "n": 100, |
| "accuracy": 0.71, |
| "soft_accuracy": 0.3801464152595509, |
| "brier": 0.04073350452343815, |
| "ece": 0.20638399999999996, |
| "score_mae": null, |
| "latency_ms_p50": 5.2139530424028635 |
| }, |
| "agent_trace.needs_review": { |
| "n": 100, |
| "accuracy": 0.87, |
| "soft_accuracy": 0.625292183322, |
| "brier": 0.029521532578379993, |
| "ece": 0.18256499999999992, |
| "score_mae": null, |
| "latency_ms_p50": 5.2139530424028635 |
| }, |
| "agent_trace.outcome": { |
| "n": 100, |
| "accuracy": 0.7, |
| "soft_accuracy": 0.407664397596501, |
| "brier": 0.06431531813416594, |
| "ece": 0.251432, |
| "score_mae": null, |
| "latency_ms_p50": 5.2139530424028635 |
| }, |
| "agent_trace.risk": { |
| "n": 100, |
| "accuracy": 0.76, |
| "soft_accuracy": 0.4461210768917738, |
| "brier": 0.04723240428914765, |
| "ece": 0.18484399999999995, |
| "score_mae": 0.24, |
| "latency_ms_p50": 5.2139530424028635 |
| }, |
| "agent_trace.urgency": { |
| "n": 100, |
| "accuracy": 0.65, |
| "soft_accuracy": 0.3780475033167836, |
| "brier": 0.046032632216030205, |
| "ece": 0.15192499999999998, |
| "score_mae": 0.35, |
| "latency_ms_p50": 5.2139530424028635 |
| }, |
| "code_review": { |
| "n": 13905, |
| "accuracy": 0.6005753326141675, |
| "soft_accuracy": 0.43762436286692236, |
| "brier": 0.20462145956578426, |
| "ece": 0.031638755843221865, |
| "score_mae": 0.8204962243797195, |
| "latency_ms_p50": 11.18671971683701 |
| }, |
| "code_review.code_quality": { |
| "n": 4635, |
| "accuracy": 0.45674217907227616, |
| "soft_accuracy": 0.3393588623455584, |
| "brier": 0.31392093132792775, |
| "ece": 0.0472170442286947, |
| "score_mae": 0.8204962243797195, |
| "latency_ms_p50": 11.18671971683701 |
| }, |
| "code_review.instruction_followed": { |
| "n": 2333, |
| "accuracy": 0.7513930561508787, |
| "soft_accuracy": 0.5743438717384003, |
| "brier": 0.11188956051875844, |
| "ece": 0.06876155165023576, |
| "score_mae": null, |
| "latency_ms_p50": 11.241163748006025 |
| }, |
| "code_review.likely_correct": { |
| "n": 2302, |
| "accuracy": 0.7528236316246742, |
| "soft_accuracy": 0.6613456443825189, |
| "brier": 0.27179493373129193, |
| "ece": 0.079262858384014, |
| "score_mae": null, |
| "latency_ms_p50": 11.151967492575448 |
| }, |
| "code_review.merge_action": { |
| "n": 4635, |
| "accuracy": 0.5928802588996763, |
| "soft_accuracy": 0.35596041263400946, |
| "brier": 0.10863594313216428, |
| "ece": 0.10759497303128371, |
| "score_mae": null, |
| "latency_ms_p50": 11.18671971683701 |
| }, |
| "routing": { |
| "n": 2121, |
| "accuracy": 0.6454502593116455, |
| "soft_accuracy": 0.4342094103303752, |
| "brier": 0.16347383356431308, |
| "ece": 0.10083290900518624, |
| "score_mae": 0.314002828854314, |
| "latency_ms_p50": 8.60158633440733 |
| }, |
| "routing.model_tier": { |
| "n": 707, |
| "accuracy": 0.4186704384724187, |
| "soft_accuracy": 0.29104883312034857, |
| "brier": 0.2916796882664165, |
| "ece": 0.03434865629420089, |
| "score_mae": null, |
| "latency_ms_p50": 8.60158633440733 |
| }, |
| "routing.skill": { |
| "n": 707, |
| "accuracy": 0.8118811881188119, |
| "soft_accuracy": 0.6114870834079437, |
| "brier": 0.09689897214124599, |
| "ece": 0.11263097595473832, |
| "score_mae": null, |
| "latency_ms_p50": 8.60158633440733 |
| }, |
| "routing.task_difficulty": { |
| "n": 707, |
| "accuracy": 0.7057991513437057, |
| "soft_accuracy": 0.40009231446283555, |
| "brier": 0.1018428402852769, |
| "ece": 0.1647117397454031, |
| "score_mae": 0.314002828854314, |
| "latency_ms_p50": 8.60158633440733 |
| }, |
| "tool_call": { |
| "n": 25031, |
| "accuracy": 0.8936119212176901, |
| "soft_accuracy": 0.7354132400184741, |
| "brier": 0.10555303215440064, |
| "ece": 0.058768491071071884, |
| "score_mae": 0.07004872955099199, |
| "latency_ms_p50": 12.551800115033984 |
| }, |
| "tool_call.call_verdict": { |
| "n": 7033, |
| "accuracy": 0.8312242286364283, |
| "soft_accuracy": 0.6229654360153449, |
| "brier": 0.132674780562995, |
| "ece": 0.08592636143893075, |
| "score_mae": null, |
| "latency_ms_p50": 12.655832339078188 |
| }, |
| "tool_call.parameter_structure": { |
| "n": 2873, |
| "accuracy": 0.9425687434737209, |
| "soft_accuracy": 0.8093260223974982, |
| "brier": 0.0475529054278188, |
| "ece": 0.06314893839192472, |
| "score_mae": 0.05986773407587887, |
| "latency_ms_p50": 11.679318360984325 |
| }, |
| "tool_call.query_coverage": { |
| "n": 2873, |
| "accuracy": 0.9644970414201184, |
| "soft_accuracy": 0.8103391947839732, |
| "brier": 0.04152526163506391, |
| "ece": 0.07684267316394014, |
| "score_mae": 0.03550295857988166, |
| "latency_ms_p50": 11.679318360984325 |
| }, |
| "tool_call.sequence_accuracy": { |
| "n": 2873, |
| "accuracy": 0.907761921336582, |
| "soft_accuracy": 0.7895875597673252, |
| "brier": 0.11586305134957572, |
| "ece": 0.053443264879916456, |
| "score_mae": 0.10442046641141664, |
| "latency_ms_p50": 11.679318360984325 |
| }, |
| "tool_call.should_call_tool": { |
| "n": 6506, |
| "accuracy": 0.8882569935444206, |
| "soft_accuracy": 0.7544548504377908, |
| "brier": 0.14932512450159993, |
| "ece": 0.05518422994159238, |
| "score_mae": null, |
| "latency_ms_p50": 13.125395790363353 |
| }, |
| "tool_call.tool_selection": { |
| "n": 2873, |
| "accuracy": 0.9244691959624086, |
| "soft_accuracy": 0.7645480035796159, |
| "brier": 0.05175455469921832, |
| "ece": 0.08405972850678739, |
| "score_mae": 0.0804037591367908, |
| "latency_ms_p50": 11.679318360984325 |
| } |
| }, |
| "majority": { |
| "ALL": { |
| "n": 41557, |
| "accuracy": 0.5345669802921289, |
| "soft_accuracy": 0.49801396628013467, |
| "brier": 0.7437008553795154, |
| "ece": 0.4654330197078711, |
| "score_mae": 0.5466126570388634, |
| "latency_ms_p50": null |
| }, |
| "agent_trace": { |
| "n": 500, |
| "accuracy": 0.36, |
| "soft_accuracy": 0.36320601275334635, |
| "brier": 0.7617543466908759, |
| "ece": 0.64, |
| "score_mae": 0.845, |
| "latency_ms_p50": null |
| }, |
| "agent_trace.action": { |
| "n": 100, |
| "accuracy": 0.34, |
| "soft_accuracy": 0.3136333298333426, |
| "brier": 0.7869731316669766, |
| "ece": 0.6599999999999999, |
| "score_mae": null, |
| "latency_ms_p50": null |
| }, |
| "agent_trace.needs_review": { |
| "n": 100, |
| "accuracy": 0.55, |
| "soft_accuracy": 0.5427667100000001, |
| "brier": 0.5737183794223801, |
| "ece": 0.44999999999999996, |
| "score_mae": null, |
| "latency_ms_p50": null |
| }, |
| "agent_trace.outcome": { |
| "n": 100, |
| "accuracy": 0.26, |
| "soft_accuracy": 0.3122366658666861, |
| "brier": 0.8482844572016827, |
| "ece": 0.74, |
| "score_mae": null, |
| "latency_ms_p50": null |
| }, |
| "agent_trace.risk": { |
| "n": 100, |
| "accuracy": 0.3, |
| "soft_accuracy": 0.30959668803336143, |
| "brier": 0.861741757986497, |
| "ece": 0.7, |
| "score_mae": 0.88, |
| "latency_ms_p50": null |
| }, |
| "agent_trace.urgency": { |
| "n": 100, |
| "accuracy": 0.35, |
| "soft_accuracy": 0.3377966700333401, |
| "brier": 0.7380540071768454, |
| "ece": 0.65, |
| "score_mae": 0.81, |
| "latency_ms_p50": null |
| }, |
| "code_review": { |
| "n": 13905, |
| "accuracy": 0.48507731032002877, |
| "soft_accuracy": 0.42916732792419915, |
| "brier": 0.75125638155265, |
| "ece": 0.5149226896799712, |
| "score_mae": 1.0017259978425026, |
| "latency_ms_p50": null |
| }, |
| "code_review.code_quality": { |
| "n": 4635, |
| "accuracy": 0.3326860841423948, |
| "soft_accuracy": 0.29555703021936686, |
| "brier": 1.0224402787502134, |
| "ece": 0.6673139158576051, |
| "score_mae": 1.0017259978425026, |
| "latency_ms_p50": null |
| }, |
| "code_review.instruction_followed": { |
| "n": 2333, |
| "accuracy": 0.5662237462494643, |
| "soft_accuracy": 0.5045862671206824, |
| "brier": 0.6618268633733213, |
| "ece": 0.43377625375053575, |
| "score_mae": null, |
| "latency_ms_p50": null |
| }, |
| "code_review.likely_correct": { |
| "n": 2302, |
| "accuracy": 0.7532580364900087, |
| "soft_accuracy": 0.7127367506516157, |
| "brier": 0.4273264986967823, |
| "ece": 0.24674196350999134, |
| "score_mae": null, |
| "latency_ms_p50": null |
| }, |
| "code_review.merge_action": { |
| "n": 4635, |
| "accuracy": 0.4634304207119741, |
| "soft_accuracy": 0.3839795250326813, |
| "brier": 0.6859679873208044, |
| "ece": 0.5365695792880258, |
| "score_mae": null, |
| "latency_ms_p50": null |
| }, |
| "routing": { |
| "n": 2121, |
| "accuracy": 0.4526166902404526, |
| "soft_accuracy": 0.3796902013726896, |
| "brier": 0.8319999746512764, |
| "ece": 0.5473833097595474, |
| "score_mae": 0.6760961810466761, |
| "latency_ms_p50": null |
| }, |
| "routing.model_tier": { |
| "n": 707, |
| "accuracy": 0.4200848656294201, |
| "soft_accuracy": 0.3065111727338155, |
| "brier": 0.9638235895227224, |
| "ece": 0.57991513437058, |
| "score_mae": null, |
| "latency_ms_p50": null |
| }, |
| "routing.skill": { |
| "n": 707, |
| "accuracy": 0.5487977369165488, |
| "soft_accuracy": 0.48265260642916186, |
| "brier": 0.737355320469855, |
| "ece": 0.45120226308345124, |
| "score_mae": null, |
| "latency_ms_p50": null |
| }, |
| "routing.task_difficulty": { |
| "n": 707, |
| "accuracy": 0.38896746817538896, |
| "soft_accuracy": 0.34990682495509523, |
| "brier": 0.7948210139612376, |
| "ece": 0.611032531824611, |
| "score_mae": 0.6760961810466761, |
| "latency_ms_p50": null |
| }, |
| "tool_call": { |
| "n": 25031, |
| "accuracy": 0.5724901122607966, |
| "soft_accuracy": 0.5489780183944137, |
| "brier": 0.7316610339955163, |
| "ece": 0.4275098877392034, |
| "score_mae": 0.3498955795335886, |
| "latency_ms_p50": null |
| }, |
| "tool_call.call_verdict": { |
| "n": 7033, |
| "accuracy": 0.5384615384615384, |
| "soft_accuracy": 0.4594399893495609, |
| "brier": 0.8238847454614278, |
| "ece": 0.46153846153846156, |
| "score_mae": null, |
| "latency_ms_p50": null |
| }, |
| "tool_call.parameter_structure": { |
| "n": 2873, |
| "accuracy": 0.8517229376957883, |
| "soft_accuracy": 0.8523165100359604, |
| "brier": 0.15079659078779242, |
| "ece": 0.14827706230421167, |
| "score_mae": 0.16498433693003828, |
| "latency_ms_p50": null |
| }, |
| "tool_call.query_coverage": { |
| "n": 2873, |
| "accuracy": 0.7514792899408284, |
| "soft_accuracy": 0.7276519317786235, |
| "brier": 0.39613735576439596, |
| "ece": 0.24852071005917165, |
| "score_mae": 0.2770623042116255, |
| "latency_ms_p50": null |
| }, |
| "tool_call.sequence_accuracy": { |
| "n": 2873, |
| "accuracy": 0.6244343891402715, |
| "soft_accuracy": 0.5982352941176617, |
| "brier": 0.6889294117646958, |
| "ece": 0.3755656108597285, |
| "score_mae": 0.5078315349808562, |
| "latency_ms_p50": null |
| }, |
| "tool_call.should_call_tool": { |
| "n": 6506, |
| "accuracy": 0.3556716876729173, |
| "soft_accuracy": 0.3472179526590886, |
| "brier": 1.1903640946817313, |
| "ece": 0.6443283123270827, |
| "score_mae": null, |
| "latency_ms_p50": null |
| }, |
| "tool_call.tool_selection": { |
| "n": 2873, |
| "accuracy": 0.6366167768882701, |
| "soft_accuracy": 0.6937862861120654, |
| "brier": 0.4262727806009997, |
| "ece": 0.3633832231117299, |
| "score_mae": 0.44970414201183434, |
| "latency_ms_p50": null |
| } |
| }, |
| "zero_shot": { |
| "ALL": { |
| "n": 41557, |
| "accuracy": 0.4495512188079024, |
| "soft_accuracy": 0.3555622636666341, |
| "brier": 0.4759449766209747, |
| "ece": 0.13287126837837185, |
| "score_mae": 0.6015028765997417, |
| "latency_ms_p50": 12.262549810111523 |
| }, |
| "agent_trace": { |
| "n": 500, |
| "accuracy": 0.384, |
| "soft_accuracy": 0.3383222443417828, |
| "brier": 0.23905442880209826, |
| "ece": 0.1748602, |
| "score_mae": 0.855, |
| "latency_ms_p50": 5.243273265659809 |
| }, |
| "agent_trace.action": { |
| "n": 100, |
| "accuracy": 0.34, |
| "soft_accuracy": 0.2753353724584661, |
| "brier": 0.22138194844624695, |
| "ece": 0.18946300000000002, |
| "score_mae": null, |
| "latency_ms_p50": 5.243273265659809 |
| }, |
| "agent_trace.needs_review": { |
| "n": 100, |
| "accuracy": 0.47, |
| "soft_accuracy": 0.5049384545179999, |
| "brier": 0.17746708898638006, |
| "ece": 0.14444700000000002, |
| "score_mae": null, |
| "latency_ms_p50": 5.243273265659809 |
| }, |
| "agent_trace.outcome": { |
| "n": 100, |
| "accuracy": 0.45, |
| "soft_accuracy": 0.31141803381489647, |
| "brier": 0.18086825047871102, |
| "ece": 0.14976499999999995, |
| "score_mae": null, |
| "latency_ms_p50": 5.243273265659809 |
| }, |
| "agent_trace.risk": { |
| "n": 100, |
| "accuracy": 0.31, |
| "soft_accuracy": 0.2871761327055934, |
| "brier": 0.25651174481546657, |
| "ece": 0.177668, |
| "score_mae": 0.9, |
| "latency_ms_p50": 5.243273265659809 |
| }, |
| "agent_trace.urgency": { |
| "n": 100, |
| "accuracy": 0.35, |
| "soft_accuracy": 0.3127432282119582, |
| "brier": 0.3590431112836864, |
| "ece": 0.38155000000000006, |
| "score_mae": 0.81, |
| "latency_ms_p50": 5.243273265659809 |
| }, |
| "code_review": { |
| "n": 13905, |
| "accuracy": 0.2746494066882416, |
| "soft_accuracy": 0.3079304536603386, |
| "brier": 0.3907150229664912, |
| "ece": 0.2118814383315354, |
| "score_mae": 1.2159654800431499, |
| "latency_ms_p50": 11.264183248082796 |
| }, |
| "code_review.code_quality": { |
| "n": 4635, |
| "accuracy": 0.2614886731391586, |
| "soft_accuracy": 0.22136276379835138, |
| "brier": 0.4098319305398359, |
| "ece": 0.039814412081984885, |
| "score_mae": 1.2159654800431499, |
| "latency_ms_p50": 11.264183248082796 |
| }, |
| "code_review.instruction_followed": { |
| "n": 2333, |
| "accuracy": 0.45906558079725673, |
| "soft_accuracy": 0.5059872505305001, |
| "brier": 0.3192942502356396, |
| "ece": 0.29432760394342045, |
| "score_mae": null, |
| "latency_ms_p50": 11.303223669528961 |
| }, |
| "code_review.likely_correct": { |
| "n": 2302, |
| "accuracy": 0.35621198957428324, |
| "soft_accuracy": 0.43913897132927865, |
| "brier": 0.6004540551520414, |
| "ece": 0.36725212858384015, |
| "score_mae": null, |
| "latency_ms_p50": 11.229163501411676 |
| }, |
| "code_review.merge_action": { |
| "n": 4635, |
| "accuracy": 0.15447680690399138, |
| "soft_accuracy": 0.22964193753052564, |
| "brier": 0.30337921802313095, |
| "ece": 0.2667025458468177, |
| "score_mae": null, |
| "latency_ms_p50": 11.264183248082796 |
| }, |
| "routing": { |
| "n": 2121, |
| "accuracy": 0.36020744931636023, |
| "soft_accuracy": 0.272360666871322, |
| "brier": 0.3998127569826348, |
| "ece": 0.113611315417256, |
| "score_mae": 0.545968882602546, |
| "latency_ms_p50": 8.667557345082363 |
| }, |
| "routing.model_tier": { |
| "n": 707, |
| "accuracy": 0.3521923620933522, |
| "soft_accuracy": 0.2723404817763805, |
| "brier": 0.4081496013531005, |
| "ece": 0.15772984441301272, |
| "score_mae": null, |
| "latency_ms_p50": 8.667557345082363 |
| }, |
| "routing.skill": { |
| "n": 707, |
| "accuracy": 0.22065063649222066, |
| "soft_accuracy": 0.20394731084366793, |
| "brier": 0.6251532200156299, |
| "ece": 0.23935954738330978, |
| "score_mae": null, |
| "latency_ms_p50": 8.667557345082363 |
| }, |
| "routing.task_difficulty": { |
| "n": 707, |
| "accuracy": 0.5077793493635078, |
| "soft_accuracy": 0.340794207993919, |
| "brier": 0.16613544957917267, |
| "ece": 0.06291711456859972, |
| "score_mae": 0.545968882602546, |
| "latency_ms_p50": 8.667557345082363 |
| }, |
| "tool_call": { |
| "n": 25031, |
| "accuracy": 0.5555910670768247, |
| "soft_accuracy": 0.3894167207239965, |
| "brier": 0.5344741691153987, |
| "ece": 0.20038942910790614, |
| "score_mae": 0.35268012530455967, |
| "latency_ms_p50": 12.63355859555304 |
| }, |
| "tool_call.call_verdict": { |
| "n": 7033, |
| "accuracy": 0.5090288639272004, |
| "soft_accuracy": 0.2763339934647179, |
| "brier": 0.4873584676179949, |
| "ece": 0.14773786435376085, |
| "score_mae": null, |
| "latency_ms_p50": 12.750410009175539 |
| }, |
| "tool_call.parameter_structure": { |
| "n": 2873, |
| "accuracy": 0.84963452836756, |
| "soft_accuracy": 0.5603033966443252, |
| "brier": 0.2091076900711842, |
| "ece": 0.23563003828750437, |
| "score_mae": 0.16916115558649494, |
| "latency_ms_p50": 11.710590030997992 |
| }, |
| "tool_call.query_coverage": { |
| "n": 2873, |
| "accuracy": 0.7434737208492864, |
| "soft_accuracy": 0.5031096529296475, |
| "brier": 0.3056886843956699, |
| "ece": 0.15529136790810996, |
| "score_mae": 0.28402366863905326, |
| "latency_ms_p50": 11.710590030997992 |
| }, |
| "tool_call.sequence_accuracy": { |
| "n": 2873, |
| "accuracy": 0.6108597285067874, |
| "soft_accuracy": 0.42309666497926895, |
| "brier": 0.4585507036080716, |
| "ece": 0.08191855203619908, |
| "score_mae": 0.5053950574312566, |
| "latency_ms_p50": 11.710590030997992 |
| }, |
| "tool_call.should_call_tool": { |
| "n": 6506, |
| "accuracy": 0.3336919766369505, |
| "soft_accuracy": 0.3170915542829213, |
| "brier": 0.9759323309997293, |
| "ece": 0.502032047340916, |
| "score_mae": null, |
| "latency_ms_p50": 13.202114030718803 |
| }, |
| "tool_call.tool_selection": { |
| "n": 2873, |
| "accuracy": 0.6348764357814132, |
| "soft_accuracy": 0.5117622340163136, |
| "brier": 0.2801910628530002, |
| "ece": 0.01890006961364427, |
| "score_mae": 0.45214061956143403, |
| "latency_ms_p50": 11.710590030997992 |
| } |
| }, |
| "typed": { |
| "ALL": { |
| "n": 41557, |
| "accuracy": 0.45051375219577927, |
| "soft_accuracy": 0.3390969306632602, |
| "brier": 0.40869457266376125, |
| "ece": 0.055596229275452984, |
| "score_mae": 0.6435364564987671, |
| "latency_ms_p50": 12.301214970648289 |
| }, |
| "agent_trace": { |
| "n": 500, |
| "accuracy": 0.714, |
| "soft_accuracy": 0.4185213375514336, |
| "brier": 0.05229190380897771, |
| "ece": 0.2114094, |
| "score_mae": 0.315, |
| "latency_ms_p50": 5.285132722929121 |
| }, |
| "agent_trace.action": { |
| "n": 100, |
| "accuracy": 0.66, |
| "soft_accuracy": 0.3378858933708014, |
| "brier": 0.056391282386058256, |
| "ece": 0.25632999999999995, |
| "score_mae": null, |
| "latency_ms_p50": 5.285132722929121 |
| }, |
| "agent_trace.needs_review": { |
| "n": 100, |
| "accuracy": 0.88, |
| "soft_accuracy": 0.6068831542299998, |
| "brier": 0.033795292162379996, |
| "ece": 0.20902199999999999, |
| "score_mae": null, |
| "latency_ms_p50": 5.285132722929121 |
| }, |
| "agent_trace.outcome": { |
| "n": 100, |
| "accuracy": 0.66, |
| "soft_accuracy": 0.3640242031319801, |
| "brier": 0.07764432812338702, |
| "ece": 0.221677, |
| "score_mae": null, |
| "latency_ms_p50": 5.285132722929121 |
| }, |
| "agent_trace.risk": { |
| "n": 100, |
| "accuracy": 0.75, |
| "soft_accuracy": 0.41817244025729494, |
| "brier": 0.04732115045857487, |
| "ece": 0.223152, |
| "score_mae": 0.25, |
| "latency_ms_p50": 5.285132722929121 |
| }, |
| "agent_trace.urgency": { |
| "n": 100, |
| "accuracy": 0.62, |
| "soft_accuracy": 0.36564099676709183, |
| "brier": 0.046307465914488456, |
| "ece": 0.14686600000000005, |
| "score_mae": 0.38, |
| "latency_ms_p50": 5.285132722929121 |
| }, |
| "code_review": { |
| "n": 13905, |
| "accuracy": 0.2575332614167566, |
| "soft_accuracy": 0.3145546199361667, |
| "brier": 0.33256056458936434, |
| "ece": 0.1680778784609853, |
| "score_mae": 1.3598705501618122, |
| "latency_ms_p50": 11.288903653621674 |
| }, |
| "code_review.code_quality": { |
| "n": 4635, |
| "accuracy": 0.2161812297734628, |
| "soft_accuracy": 0.2156468273418538, |
| "brier": 0.4133553874242992, |
| "ece": 0.0773525134843581, |
| "score_mae": 1.3598705501618122, |
| "latency_ms_p50": 11.288903653621674 |
| }, |
| "code_review.instruction_followed": { |
| "n": 2333, |
| "accuracy": 0.45049292756108017, |
| "soft_accuracy": 0.5031287105466815, |
| "brier": 0.22465357059209978, |
| "ece": 0.20252091727389623, |
| "score_mae": null, |
| "latency_ms_p50": 11.332182368884483 |
| }, |
| "code_review.likely_correct": { |
| "n": 2302, |
| "accuracy": 0.32493483927019984, |
| "soft_accuracy": 0.4597893605560382, |
| "brier": 0.4740768352997398, |
| "ece": 0.2952202867072111, |
| "score_mae": null, |
| "latency_ms_p50": 11.257084629808864 |
| }, |
| "code_review.merge_action": { |
| "n": 4635, |
| "accuracy": 0.16828478964401294, |
| "soft_accuracy": 0.2464130648926627, |
| "brier": 0.2357952049303329, |
| "ece": 0.1784133764832794, |
| "score_mae": null, |
| "latency_ms_p50": 11.288903653621674 |
| }, |
| "routing": { |
| "n": 2121, |
| "accuracy": 0.35454974068835454, |
| "soft_accuracy": 0.26241057114053784, |
| "brier": 0.3317003871036206, |
| "ece": 0.026113531353135305, |
| "score_mae": 0.5318246110325319, |
| "latency_ms_p50": 8.731171023100615 |
| }, |
| "routing.model_tier": { |
| "n": 707, |
| "accuracy": 0.272984441301273, |
| "soft_accuracy": 0.2581747552619435, |
| "brier": 0.3383839440346083, |
| "ece": 0.08653380480905236, |
| "score_mae": null, |
| "latency_ms_p50": 8.731171023100615 |
| }, |
| "routing.skill": { |
| "n": 707, |
| "accuracy": 0.2871287128712871, |
| "soft_accuracy": 0.20951270931679064, |
| "brier": 0.4897188027773141, |
| "ece": 0.0719220650636492, |
| "score_mae": null, |
| "latency_ms_p50": 8.731171023100615 |
| }, |
| "routing.task_difficulty": { |
| "n": 707, |
| "accuracy": 0.5035360678925035, |
| "soft_accuracy": 0.3195442488428794, |
| "brier": 0.1669984144989407, |
| "ece": 0.08971980198019801, |
| "score_mae": 0.5318246110325319, |
| "latency_ms_p50": 8.731171023100615 |
| }, |
| "tool_call": { |
| "n": 25031, |
| "accuracy": 0.5605848747553034, |
| "soft_accuracy": 0.35764195066900195, |
| "brier": 0.464631186633436, |
| "ece": 0.09784442091806163, |
| "score_mae": 0.3672119735468152, |
| "latency_ms_p50": 12.692837044596672 |
| }, |
| "tool_call.call_verdict": { |
| "n": 7033, |
| "accuracy": 0.5292193942840893, |
| "soft_accuracy": 0.27012174415225865, |
| "brier": 0.4859364810128098, |
| "ece": 0.15381443196360017, |
| "score_mae": null, |
| "latency_ms_p50": 12.818309012800455 |
| }, |
| "tool_call.parameter_structure": { |
| "n": 2873, |
| "accuracy": 0.8419770274973895, |
| "soft_accuracy": 0.495495654291868, |
| "brier": 0.27895918554296956, |
| "ece": 0.31188656456665514, |
| "score_mae": 0.17577445179255133, |
| "latency_ms_p50": 11.742415837943554 |
| }, |
| "tool_call.query_coverage": { |
| "n": 2873, |
| "accuracy": 0.7076226940480335, |
| "soft_accuracy": 0.4637026902444456, |
| "brier": 0.3405009573564468, |
| "ece": 0.17881566306996174, |
| "score_mae": 0.3198746954403063, |
| "latency_ms_p50": 11.742415837943554 |
| }, |
| "tool_call.sequence_accuracy": { |
| "n": 2873, |
| "accuracy": 0.5767490428123913, |
| "soft_accuracy": 0.39998245191225323, |
| "brier": 0.47585696271728206, |
| "ece": 0.10643915767490429, |
| "score_mae": 0.5248868778280543, |
| "latency_ms_p50": 11.742415837943554 |
| }, |
| "tool_call.should_call_tool": { |
| "n": 6506, |
| "accuracy": 0.3647402397786658, |
| "soft_accuracy": 0.2743444411062642, |
| "brier": 0.6462853975060024, |
| "ece": 0.07603150937596063, |
| "score_mae": null, |
| "latency_ms_p50": 13.254288118332624 |
| }, |
| "tool_call.tool_selection": { |
| "n": 2873, |
| "accuracy": 0.6362687086668987, |
| "soft_accuracy": 0.4742631669122307, |
| "brier": 0.2996913056204823, |
| "ece": 0.07137298990602153, |
| "score_mae": 0.44831186912634874, |
| "latency_ms_p50": 11.742415837943554 |
| } |
| } |
| } |
| } |
