| { | |
| "calibration": { | |
| "split": "calibration", | |
| "temperature": 1.0087398771573446, | |
| "default_prediction": "raw", | |
| "calibration_comparison": { | |
| "raw": { | |
| "count": 13766, | |
| "top1_accuracy": 0.8311782652912973, | |
| "nll": 0.6855602746553805, | |
| "brier_score": 0.010973403472421008, | |
| "ece": 0.1712850628941104, | |
| "signed_calibration_error": -0.17128506289411038, | |
| "kl_divergence": 0.02185337568727864, | |
| "confidence_analysis": { | |
| "correct": { | |
| "count": 11442, | |
| "mean": 0.694725726679517, | |
| "median": 0.6793243885040283, | |
| "std": 0.19659272893691354, | |
| "p25": 0.501576766371727, | |
| "p75": 0.8654205501079559 | |
| }, | |
| "incorrect": { | |
| "count": 2324, | |
| "mean": 0.48839847656309915, | |
| "median": 0.500387966632843, | |
| "std": 0.07211813579079812, | |
| "p25": 0.5000417530536652, | |
| "p75": 0.5010467320680618 | |
| } | |
| }, | |
| "confidence_histogram": [ | |
| { | |
| "bucket": "0.0-0.1", | |
| "sample_count": 0, | |
| "average_confidence": null, | |
| "empirical_accuracy": null, | |
| "signed_gap": null | |
| }, | |
| { | |
| "bucket": "0.1-0.2", | |
| "sample_count": 0, | |
| "average_confidence": null, | |
| "empirical_accuracy": null, | |
| "signed_gap": null | |
| }, | |
| { | |
| "bucket": "0.2-0.3", | |
| "sample_count": 134, | |
| "average_confidence": 0.27603658633445627, | |
| "empirical_accuracy": 0.5, | |
| "signed_gap": -0.22396341366554373 | |
| }, | |
| { | |
| "bucket": "0.3-0.4", | |
| "sample_count": 637, | |
| "average_confidence": 0.3581763524649843, | |
| "empirical_accuracy": 0.6923076923076923, | |
| "signed_gap": -0.334131339842708 | |
| }, | |
| { | |
| "bucket": "0.4-0.5", | |
| "sample_count": 1197, | |
| "average_confidence": 0.45263352763583087, | |
| "empirical_accuracy": 0.7861319966583125, | |
| "signed_gap": -0.3334984690224816 | |
| }, | |
| { | |
| "bucket": "0.5-0.6", | |
| "sample_count": 4710, | |
| "average_confidence": 0.520006962819717, | |
| "empirical_accuracy": 0.6322717622080679, | |
| "signed_gap": -0.11226479938835088 | |
| }, | |
| { | |
| "bucket": "0.6-0.7", | |
| "sample_count": 1701, | |
| "average_confidence": 0.6498155774683338, | |
| "empirical_accuracy": 0.9711934156378601, | |
| "signed_gap": -0.32137783816952625 | |
| }, | |
| { | |
| "bucket": "0.7-0.8", | |
| "sample_count": 1530, | |
| "average_confidence": 0.7490799561045528, | |
| "empirical_accuracy": 0.9888888888888889, | |
| "signed_gap": -0.23980893278433613 | |
| }, | |
| { | |
| "bucket": "0.8-0.9", | |
| "sample_count": 1452, | |
| "average_confidence": 0.847257420872197, | |
| "empirical_accuracy": 0.9986225895316805, | |
| "signed_gap": -0.15136516865948346 | |
| }, | |
| { | |
| "bucket": "0.9-1.0", | |
| "sample_count": 2405, | |
| "average_confidence": 0.9755763211765804, | |
| "empirical_accuracy": 0.997920997920998, | |
| "signed_gap": -0.022344676744417535 | |
| } | |
| ], | |
| "selective_accuracy": [ | |
| { | |
| "threshold": 0.5, | |
| "coverage": 0.8570390817957286, | |
| "accuracy": 0.8470079674521105, | |
| "risk": 0.1529920325478895, | |
| "sample_count": 11798 | |
| }, | |
| { | |
| "threshold": 0.6, | |
| "coverage": 0.514891762312945, | |
| "accuracy": 0.9897009029345373, | |
| "risk": 0.010299097065462726, | |
| "sample_count": 7088 | |
| }, | |
| { | |
| "threshold": 0.7, | |
| "coverage": 0.39132645648699693, | |
| "accuracy": 0.9955448301466493, | |
| "risk": 0.0044551698533507045, | |
| "sample_count": 5387 | |
| }, | |
| { | |
| "threshold": 0.8, | |
| "coverage": 0.28018305971233476, | |
| "accuracy": 0.9981851179673321, | |
| "risk": 0.001814882032667886, | |
| "sample_count": 3857 | |
| }, | |
| { | |
| "threshold": 0.9, | |
| "coverage": 0.1747057968908906, | |
| "accuracy": 0.997920997920998, | |
| "risk": 0.0020790020790020236, | |
| "sample_count": 2405 | |
| }, | |
| { | |
| "threshold": 0.95, | |
| "coverage": 0.13162865029783524, | |
| "accuracy": 0.9977924944812362, | |
| "risk": 0.002207505518763808, | |
| "sample_count": 1812 | |
| } | |
| ] | |
| }, | |
| "temperature_scaled": { | |
| "count": 13766, | |
| "top1_accuracy": 0.8311782652912973, | |
| "nll": 0.6855494345867709, | |
| "brier_score": 0.010966401509070522, | |
| "ece": 0.17250904779846635, | |
| "signed_calibration_error": -0.17250904779846632, | |
| "kl_divergence": 0.02184253561866892, | |
| "confidence_analysis": { | |
| "correct": { | |
| "count": 11442, | |
| "mean": 0.6933398573451267, | |
| "median": 0.6774639946027864, | |
| "std": 0.19661340445165715, | |
| "p25": 0.5014006484541963, | |
| "p75": 0.863254437984726 | |
| }, | |
| "incorrect": { | |
| "count": 2324, | |
| "mean": 0.4879715147432757, | |
| "median": 0.5003803821956974, | |
| "std": 0.07215117413704505, | |
| "p25": 0.5000357963828128, | |
| "p75": 0.5010135416360336 | |
| } | |
| }, | |
| "confidence_histogram": [ | |
| { | |
| "bucket": "0.0-0.1", | |
| "sample_count": 0, | |
| "average_confidence": null, | |
| "empirical_accuracy": null, | |
| "signed_gap": null | |
| }, | |
| { | |
| "bucket": "0.1-0.2", | |
| "sample_count": 0, | |
| "average_confidence": null, | |
| "empirical_accuracy": null, | |
| "signed_gap": null | |
| }, | |
| { | |
| "bucket": "0.2-0.3", | |
| "sample_count": 137, | |
| "average_confidence": 0.2758391267365239, | |
| "empirical_accuracy": 0.5036496350364964, | |
| "signed_gap": -0.22781050829997251 | |
| }, | |
| { | |
| "bucket": "0.3-0.4", | |
| "sample_count": 642, | |
| "average_confidence": 0.3577477496059755, | |
| "empirical_accuracy": 0.6962616822429907, | |
| "signed_gap": -0.3385139326370152 | |
| }, | |
| { | |
| "bucket": "0.4-0.5", | |
| "sample_count": 1215, | |
| "average_confidence": 0.4524140225007448, | |
| "empirical_accuracy": 0.7851851851851852, | |
| "signed_gap": -0.3327711626844404 | |
| }, | |
| { | |
| "bucket": "0.5-0.6", | |
| "sample_count": 4721, | |
| "average_confidence": 0.5201900803979108, | |
| "empirical_accuracy": 0.633552213514086, | |
| "signed_gap": -0.11336213311617516 | |
| }, | |
| { | |
| "bucket": "0.6-0.7", | |
| "sample_count": 1701, | |
| "average_confidence": 0.6500267694102276, | |
| "empirical_accuracy": 0.9723691945914168, | |
| "signed_gap": -0.32234242518118916 | |
| }, | |
| { | |
| "bucket": "0.7-0.8", | |
| "sample_count": 1529, | |
| "average_confidence": 0.7491738888998721, | |
| "empirical_accuracy": 0.9895356442119032, | |
| "signed_gap": -0.24036175531203108 | |
| }, | |
| { | |
| "bucket": "0.8-0.9", | |
| "sample_count": 1445, | |
| "average_confidence": 0.8472853994689213, | |
| "empirical_accuracy": 0.9986159169550173, | |
| "signed_gap": -0.15133051748609594 | |
| }, | |
| { | |
| "bucket": "0.9-1.0", | |
| "sample_count": 2376, | |
| "average_confidence": 0.9759117055438289, | |
| "empirical_accuracy": 0.997895622895623, | |
| "signed_gap": -0.021983917351794058 | |
| } | |
| ], | |
| "selective_accuracy": [ | |
| { | |
| "threshold": 0.5, | |
| "coverage": 0.8551503704779893, | |
| "accuracy": 0.8470948012232415, | |
| "risk": 0.15290519877675846, | |
| "sample_count": 11772 | |
| }, | |
| { | |
| "threshold": 0.6, | |
| "coverage": 0.5122039808223159, | |
| "accuracy": 0.9900723301659339, | |
| "risk": 0.009927669834066122, | |
| "sample_count": 7051 | |
| }, | |
| { | |
| "threshold": 0.7, | |
| "coverage": 0.38863867499636784, | |
| "accuracy": 0.9957009345794392, | |
| "risk": 0.004299065420560799, | |
| "sample_count": 5350 | |
| }, | |
| { | |
| "threshold": 0.8, | |
| "coverage": 0.2775679209646956, | |
| "accuracy": 0.9981680188432348, | |
| "risk": 0.0018319811567651945, | |
| "sample_count": 3821 | |
| }, | |
| { | |
| "threshold": 0.9, | |
| "coverage": 0.17259915734418133, | |
| "accuracy": 0.997895622895623, | |
| "risk": 0.002104377104377053, | |
| "sample_count": 2376 | |
| }, | |
| { | |
| "threshold": 0.95, | |
| "coverage": 0.1306116518959756, | |
| "accuracy": 0.9977753058954394, | |
| "risk": 0.0022246941045606095, | |
| "sample_count": 1798 | |
| } | |
| ] | |
| } | |
| }, | |
| "overall": { | |
| "count": 13766, | |
| "top1_accuracy": 0.8311782652912973, | |
| "nll": 0.6855602746553805, | |
| "brier_score": 0.010973403472421008, | |
| "ece": 0.1712850628941104, | |
| "signed_calibration_error": -0.17128506289411038, | |
| "kl_divergence": 0.02185337568727864, | |
| "confidence_analysis": { | |
| "correct": { | |
| "count": 11442, | |
| "mean": 0.694725726679517, | |
| "median": 0.6793243885040283, | |
| "std": 0.19659272893691354, | |
| "p25": 0.501576766371727, | |
| "p75": 0.8654205501079559 | |
| }, | |
| "incorrect": { | |
| "count": 2324, | |
| "mean": 0.48839847656309915, | |
| "median": 0.500387966632843, | |
| "std": 0.07211813579079812, | |
| "p25": 0.5000417530536652, | |
| "p75": 0.5010467320680618 | |
| } | |
| }, | |
| "confidence_histogram": [ | |
| { | |
| "bucket": "0.0-0.1", | |
| "sample_count": 0, | |
| "average_confidence": null, | |
| "empirical_accuracy": null, | |
| "signed_gap": null | |
| }, | |
| { | |
| "bucket": "0.1-0.2", | |
| "sample_count": 0, | |
| "average_confidence": null, | |
| "empirical_accuracy": null, | |
| "signed_gap": null | |
| }, | |
| { | |
| "bucket": "0.2-0.3", | |
| "sample_count": 134, | |
| "average_confidence": 0.27603658633445627, | |
| "empirical_accuracy": 0.5, | |
| "signed_gap": -0.22396341366554373 | |
| }, | |
| { | |
| "bucket": "0.3-0.4", | |
| "sample_count": 637, | |
| "average_confidence": 0.3581763524649843, | |
| "empirical_accuracy": 0.6923076923076923, | |
| "signed_gap": -0.334131339842708 | |
| }, | |
| { | |
| "bucket": "0.4-0.5", | |
| "sample_count": 1197, | |
| "average_confidence": 0.45263352763583087, | |
| "empirical_accuracy": 0.7861319966583125, | |
| "signed_gap": -0.3334984690224816 | |
| }, | |
| { | |
| "bucket": "0.5-0.6", | |
| "sample_count": 4710, | |
| "average_confidence": 0.520006962819717, | |
| "empirical_accuracy": 0.6322717622080679, | |
| "signed_gap": -0.11226479938835088 | |
| }, | |
| { | |
| "bucket": "0.6-0.7", | |
| "sample_count": 1701, | |
| "average_confidence": 0.6498155774683338, | |
| "empirical_accuracy": 0.9711934156378601, | |
| "signed_gap": -0.32137783816952625 | |
| }, | |
| { | |
| "bucket": "0.7-0.8", | |
| "sample_count": 1530, | |
| "average_confidence": 0.7490799561045528, | |
| "empirical_accuracy": 0.9888888888888889, | |
| "signed_gap": -0.23980893278433613 | |
| }, | |
| { | |
| "bucket": "0.8-0.9", | |
| "sample_count": 1452, | |
| "average_confidence": 0.847257420872197, | |
| "empirical_accuracy": 0.9986225895316805, | |
| "signed_gap": -0.15136516865948346 | |
| }, | |
| { | |
| "bucket": "0.9-1.0", | |
| "sample_count": 2405, | |
| "average_confidence": 0.9755763211765804, | |
| "empirical_accuracy": 0.997920997920998, | |
| "signed_gap": -0.022344676744417535 | |
| } | |
| ], | |
| "selective_accuracy": [ | |
| { | |
| "threshold": 0.5, | |
| "coverage": 0.8570390817957286, | |
| "accuracy": 0.8470079674521105, | |
| "risk": 0.1529920325478895, | |
| "sample_count": 11798 | |
| }, | |
| { | |
| "threshold": 0.6, | |
| "coverage": 0.514891762312945, | |
| "accuracy": 0.9897009029345373, | |
| "risk": 0.010299097065462726, | |
| "sample_count": 7088 | |
| }, | |
| { | |
| "threshold": 0.7, | |
| "coverage": 0.39132645648699693, | |
| "accuracy": 0.9955448301466493, | |
| "risk": 0.0044551698533507045, | |
| "sample_count": 5387 | |
| }, | |
| { | |
| "threshold": 0.8, | |
| "coverage": 0.28018305971233476, | |
| "accuracy": 0.9981851179673321, | |
| "risk": 0.001814882032667886, | |
| "sample_count": 3857 | |
| }, | |
| { | |
| "threshold": 0.9, | |
| "coverage": 0.1747057968908906, | |
| "accuracy": 0.997920997920998, | |
| "risk": 0.0020790020790020236, | |
| "sample_count": 2405 | |
| }, | |
| { | |
| "threshold": 0.95, | |
| "coverage": 0.13162865029783524, | |
| "accuracy": 0.9977924944812362, | |
| "risk": 0.002207505518763808, | |
| "sample_count": 1812 | |
| } | |
| ] | |
| }, | |
| "by_kind": { | |
| "noul": { | |
| "count": 7175, | |
| "top1_accuracy": 0.7839721254355401, | |
| "nll": 0.5094520074091426, | |
| "brier_score": 0.0037153609942311735, | |
| "ece": 0.0968418979893994, | |
| "signed_calibration_error": -0.09684189798939934, | |
| "kl_divergence": 0.005356521734346802 | |
| }, | |
| "choice": { | |
| "count": 3418, | |
| "top1_accuracy": 0.8838502047981276, | |
| "nll": 0.7117832643875154, | |
| "brier_score": 0.0267761779649997, | |
| "ece": 0.18598052990966682, | |
| "signed_calibration_error": -0.18598052990966682, | |
| "kl_divergence": 0.05244051993410699 | |
| }, | |
| "score": { | |
| "count": 3173, | |
| "top1_accuracy": 0.8811849984242042, | |
| "nll": 1.055540305726077, | |
| "brier_score": 0.010362805163369038, | |
| "ece": 0.323790767883751, | |
| "signed_calibration_error": -0.323790767883751, | |
| "kl_divergence": 0.026208266351201326 | |
| } | |
| } | |
| }, | |
| "validation": { | |
| "split": "validation", | |
| "temperature": 1.0087398771573446, | |
| "default_prediction": "raw", | |
| "calibration_comparison": { | |
| "raw": { | |
| "count": 14111, | |
| "top1_accuracy": 0.8263765856424067, | |
| "nll": 0.6840500261181026, | |
| "brier_score": 0.011582265527708373, | |
| "ece": 0.16699567031458232, | |
| "signed_calibration_error": -0.1669956703145823, | |
| "kl_divergence": 0.02310311772478613, | |
| "confidence_analysis": { | |
| "correct": { | |
| "count": 11661, | |
| "mean": 0.6960411451404903, | |
| "median": 0.6765217781066895, | |
| "std": 0.19690960502309285, | |
| "p25": 0.50345379114151, | |
| "p75": 0.8713327646255493 | |
| }, | |
| "incorrect": { | |
| "count": 2450, | |
| "mean": 0.4848931847786417, | |
| "median": 0.500392496585846, | |
| "std": 0.07416498496851862, | |
| "p25": 0.5000284314155579, | |
| "p75": 0.5010085254907608 | |
| } | |
| }, | |
| "confidence_histogram": [ | |
| { | |
| "bucket": "0.0-0.1", | |
| "sample_count": 0, | |
| "average_confidence": null, | |
| "empirical_accuracy": null, | |
| "signed_gap": null | |
| }, | |
| { | |
| "bucket": "0.1-0.2", | |
| "sample_count": 0, | |
| "average_confidence": null, | |
| "empirical_accuracy": null, | |
| "signed_gap": null | |
| }, | |
| { | |
| "bucket": "0.2-0.3", | |
| "sample_count": 148, | |
| "average_confidence": 0.27538005906987834, | |
| "empirical_accuracy": 0.46621621621621623, | |
| "signed_gap": -0.1908361571463379 | |
| }, | |
| { | |
| "bucket": "0.3-0.4", | |
| "sample_count": 667, | |
| "average_confidence": 0.3564207344755776, | |
| "empirical_accuracy": 0.664167916041979, | |
| "signed_gap": -0.3077471815664014 | |
| }, | |
| { | |
| "bucket": "0.4-0.5", | |
| "sample_count": 1160, | |
| "average_confidence": 0.4550788116095395, | |
| "empirical_accuracy": 0.7741379310344828, | |
| "signed_gap": -0.3190591194249433 | |
| }, | |
| { | |
| "bucket": "0.5-0.6", | |
| "sample_count": 4972, | |
| "average_confidence": 0.5200908452703985, | |
| "empirical_accuracy": 0.6351568785197104, | |
| "signed_gap": -0.11506603324931186 | |
| }, | |
| { | |
| "bucket": "0.6-0.7", | |
| "sample_count": 1678, | |
| "average_confidence": 0.6489259644685684, | |
| "empirical_accuracy": 0.9713945172824792, | |
| "signed_gap": -0.32246855281391074 | |
| }, | |
| { | |
| "bucket": "0.7-0.8", | |
| "sample_count": 1540, | |
| "average_confidence": 0.7498124199641215, | |
| "empirical_accuracy": 0.9935064935064936, | |
| "signed_gap": -0.24369407354237205 | |
| }, | |
| { | |
| "bucket": "0.8-0.9", | |
| "sample_count": 1412, | |
| "average_confidence": 0.8490157142154894, | |
| "empirical_accuracy": 0.9936260623229461, | |
| "signed_gap": -0.14461034810745677 | |
| }, | |
| { | |
| "bucket": "0.9-1.0", | |
| "sample_count": 2534, | |
| "average_confidence": 0.9746775966087663, | |
| "empirical_accuracy": 0.9984214680347278, | |
| "signed_gap": -0.02374387142596146 | |
| } | |
| ], | |
| "selective_accuracy": [ | |
| { | |
| "threshold": 0.5, | |
| "coverage": 0.8600382680178584, | |
| "accuracy": 0.844676994067238, | |
| "risk": 0.15532300593276205, | |
| "sample_count": 12136 | |
| }, | |
| { | |
| "threshold": 0.6, | |
| "coverage": 0.5076890369215505, | |
| "accuracy": 0.9900893355667225, | |
| "risk": 0.009910664433277483, | |
| "sample_count": 7164 | |
| }, | |
| { | |
| "threshold": 0.7, | |
| "coverage": 0.3887747147615336, | |
| "accuracy": 0.9958075100255195, | |
| "risk": 0.004192489974480518, | |
| "sample_count": 5486 | |
| }, | |
| { | |
| "threshold": 0.8, | |
| "coverage": 0.279639997165332, | |
| "accuracy": 0.996705524581855, | |
| "risk": 0.003294475418144982, | |
| "sample_count": 3946 | |
| }, | |
| { | |
| "threshold": 0.9, | |
| "coverage": 0.17957621713556798, | |
| "accuracy": 0.9984214680347278, | |
| "risk": 0.0015785319652722452, | |
| "sample_count": 2534 | |
| }, | |
| { | |
| "threshold": 0.95, | |
| "coverage": 0.13273332860888667, | |
| "accuracy": 0.9983982915109451, | |
| "risk": 0.0016017084890549427, | |
| "sample_count": 1873 | |
| } | |
| ] | |
| }, | |
| "temperature_scaled": { | |
| "count": 14111, | |
| "top1_accuracy": 0.8263057189426688, | |
| "nll": 0.6840468913421749, | |
| "brier_score": 0.011582336771387319, | |
| "ece": 0.16813608304214336, | |
| "signed_calibration_error": -0.16813608304214334, | |
| "kl_divergence": 0.023099982948858303, | |
| "confidence_analysis": { | |
| "correct": { | |
| "count": 11660, | |
| "mean": 0.694681208951386, | |
| "median": 0.6747364841480057, | |
| "std": 0.19693845421787254, | |
| "p25": 0.5021719095333224, | |
| "p75": 0.8689303683324948 | |
| }, | |
| "incorrect": { | |
| "count": 2451, | |
| "mean": 0.4844752492122222, | |
| "median": 0.5003890293824186, | |
| "std": 0.07420726284342961, | |
| "p25": 0.500027128879432, | |
| "p75": 0.5009866390613054 | |
| } | |
| }, | |
| "confidence_histogram": [ | |
| { | |
| "bucket": "0.0-0.1", | |
| "sample_count": 0, | |
| "average_confidence": null, | |
| "empirical_accuracy": null, | |
| "signed_gap": null | |
| }, | |
| { | |
| "bucket": "0.1-0.2", | |
| "sample_count": 0, | |
| "average_confidence": null, | |
| "empirical_accuracy": null, | |
| "signed_gap": null | |
| }, | |
| { | |
| "bucket": "0.2-0.3", | |
| "sample_count": 151, | |
| "average_confidence": 0.2751785209954829, | |
| "empirical_accuracy": 0.46357615894039733, | |
| "signed_gap": -0.18839763794491443 | |
| }, | |
| { | |
| "bucket": "0.3-0.4", | |
| "sample_count": 674, | |
| "average_confidence": 0.35612814661984515, | |
| "empirical_accuracy": 0.6646884272997032, | |
| "signed_gap": -0.3085602806798581 | |
| }, | |
| { | |
| "bucket": "0.4-0.5", | |
| "sample_count": 1168, | |
| "average_confidence": 0.45463179276212934, | |
| "empirical_accuracy": 0.7782534246575342, | |
| "signed_gap": -0.3236216318954049 | |
| }, | |
| { | |
| "bucket": "0.5-0.6", | |
| "sample_count": 4987, | |
| "average_confidence": 0.5201110232818913, | |
| "empirical_accuracy": 0.6360537397232805, | |
| "signed_gap": -0.1159427164413892 | |
| }, | |
| { | |
| "bucket": "0.6-0.7", | |
| "sample_count": 1673, | |
| "average_confidence": 0.6487813857971743, | |
| "empirical_accuracy": 0.9719067543335326, | |
| "signed_gap": -0.3231253685363583 | |
| }, | |
| { | |
| "bucket": "0.7-0.8", | |
| "sample_count": 1546, | |
| "average_confidence": 0.7495579579669318, | |
| "empirical_accuracy": 0.9935316946959897, | |
| "signed_gap": -0.24397373672905787 | |
| }, | |
| { | |
| "bucket": "0.8-0.9", | |
| "sample_count": 1409, | |
| "average_confidence": 0.8491268025105129, | |
| "empirical_accuracy": 0.99361249112846, | |
| "signed_gap": -0.14448568861794708 | |
| }, | |
| { | |
| "bucket": "0.9-1.0", | |
| "sample_count": 2503, | |
| "average_confidence": 0.9749893213960854, | |
| "empirical_accuracy": 0.9984019176987615, | |
| "signed_gap": -0.023412596302676114 | |
| } | |
| ], | |
| "selective_accuracy": [ | |
| { | |
| "threshold": 0.5, | |
| "coverage": 0.8587626674225781, | |
| "accuracy": 0.8444462782637399, | |
| "risk": 0.15555372173626014, | |
| "sample_count": 12118 | |
| }, | |
| { | |
| "threshold": 0.6, | |
| "coverage": 0.5053504358302033, | |
| "accuracy": 0.9901837049502173, | |
| "risk": 0.009816295049782653, | |
| "sample_count": 7131 | |
| }, | |
| { | |
| "threshold": 0.7, | |
| "coverage": 0.38679044716887534, | |
| "accuracy": 0.9957860021986076, | |
| "risk": 0.004213997801392422, | |
| "sample_count": 5458 | |
| }, | |
| { | |
| "threshold": 0.8, | |
| "coverage": 0.277230529374247, | |
| "accuracy": 0.996676891615542, | |
| "risk": 0.00332310838445804, | |
| "sample_count": 3912 | |
| }, | |
| { | |
| "threshold": 0.9, | |
| "coverage": 0.1773793494436964, | |
| "accuracy": 0.9984019176987615, | |
| "risk": 0.0015980823012384793, | |
| "sample_count": 2503 | |
| }, | |
| { | |
| "threshold": 0.95, | |
| "coverage": 0.131599461413082, | |
| "accuracy": 0.9983844911147012, | |
| "risk": 0.001615508885298822, | |
| "sample_count": 1857 | |
| } | |
| ] | |
| } | |
| }, | |
| "overall": { | |
| "count": 14111, | |
| "top1_accuracy": 0.8263765856424067, | |
| "nll": 0.6840500261181026, | |
| "brier_score": 0.011582265527708373, | |
| "ece": 0.16699567031458232, | |
| "signed_calibration_error": -0.1669956703145823, | |
| "kl_divergence": 0.02310311772478613, | |
| "confidence_analysis": { | |
| "correct": { | |
| "count": 11661, | |
| "mean": 0.6960411451404903, | |
| "median": 0.6765217781066895, | |
| "std": 0.19690960502309285, | |
| "p25": 0.50345379114151, | |
| "p75": 0.8713327646255493 | |
| }, | |
| "incorrect": { | |
| "count": 2450, | |
| "mean": 0.4848931847786417, | |
| "median": 0.500392496585846, | |
| "std": 0.07416498496851862, | |
| "p25": 0.5000284314155579, | |
| "p75": 0.5010085254907608 | |
| } | |
| }, | |
| "confidence_histogram": [ | |
| { | |
| "bucket": "0.0-0.1", | |
| "sample_count": 0, | |
| "average_confidence": null, | |
| "empirical_accuracy": null, | |
| "signed_gap": null | |
| }, | |
| { | |
| "bucket": "0.1-0.2", | |
| "sample_count": 0, | |
| "average_confidence": null, | |
| "empirical_accuracy": null, | |
| "signed_gap": null | |
| }, | |
| { | |
| "bucket": "0.2-0.3", | |
| "sample_count": 148, | |
| "average_confidence": 0.27538005906987834, | |
| "empirical_accuracy": 0.46621621621621623, | |
| "signed_gap": -0.1908361571463379 | |
| }, | |
| { | |
| "bucket": "0.3-0.4", | |
| "sample_count": 667, | |
| "average_confidence": 0.3564207344755776, | |
| "empirical_accuracy": 0.664167916041979, | |
| "signed_gap": -0.3077471815664014 | |
| }, | |
| { | |
| "bucket": "0.4-0.5", | |
| "sample_count": 1160, | |
| "average_confidence": 0.4550788116095395, | |
| "empirical_accuracy": 0.7741379310344828, | |
| "signed_gap": -0.3190591194249433 | |
| }, | |
| { | |
| "bucket": "0.5-0.6", | |
| "sample_count": 4972, | |
| "average_confidence": 0.5200908452703985, | |
| "empirical_accuracy": 0.6351568785197104, | |
| "signed_gap": -0.11506603324931186 | |
| }, | |
| { | |
| "bucket": "0.6-0.7", | |
| "sample_count": 1678, | |
| "average_confidence": 0.6489259644685684, | |
| "empirical_accuracy": 0.9713945172824792, | |
| "signed_gap": -0.32246855281391074 | |
| }, | |
| { | |
| "bucket": "0.7-0.8", | |
| "sample_count": 1540, | |
| "average_confidence": 0.7498124199641215, | |
| "empirical_accuracy": 0.9935064935064936, | |
| "signed_gap": -0.24369407354237205 | |
| }, | |
| { | |
| "bucket": "0.8-0.9", | |
| "sample_count": 1412, | |
| "average_confidence": 0.8490157142154894, | |
| "empirical_accuracy": 0.9936260623229461, | |
| "signed_gap": -0.14461034810745677 | |
| }, | |
| { | |
| "bucket": "0.9-1.0", | |
| "sample_count": 2534, | |
| "average_confidence": 0.9746775966087663, | |
| "empirical_accuracy": 0.9984214680347278, | |
| "signed_gap": -0.02374387142596146 | |
| } | |
| ], | |
| "selective_accuracy": [ | |
| { | |
| "threshold": 0.5, | |
| "coverage": 0.8600382680178584, | |
| "accuracy": 0.844676994067238, | |
| "risk": 0.15532300593276205, | |
| "sample_count": 12136 | |
| }, | |
| { | |
| "threshold": 0.6, | |
| "coverage": 0.5076890369215505, | |
| "accuracy": 0.9900893355667225, | |
| "risk": 0.009910664433277483, | |
| "sample_count": 7164 | |
| }, | |
| { | |
| "threshold": 0.7, | |
| "coverage": 0.3887747147615336, | |
| "accuracy": 0.9958075100255195, | |
| "risk": 0.004192489974480518, | |
| "sample_count": 5486 | |
| }, | |
| { | |
| "threshold": 0.8, | |
| "coverage": 0.279639997165332, | |
| "accuracy": 0.996705524581855, | |
| "risk": 0.003294475418144982, | |
| "sample_count": 3946 | |
| }, | |
| { | |
| "threshold": 0.9, | |
| "coverage": 0.17957621713556798, | |
| "accuracy": 0.9984214680347278, | |
| "risk": 0.0015785319652722452, | |
| "sample_count": 2534 | |
| }, | |
| { | |
| "threshold": 0.95, | |
| "coverage": 0.13273332860888667, | |
| "accuracy": 0.9983982915109451, | |
| "risk": 0.0016017084890549427, | |
| "sample_count": 1873 | |
| } | |
| ] | |
| }, | |
| "by_kind": { | |
| "noul": { | |
| "count": 7318, | |
| "top1_accuracy": 0.7768515987974857, | |
| "nll": 0.5093834824517124, | |
| "brier_score": 0.004471756349480391, | |
| "ece": 0.0906974503693603, | |
| "signed_calibration_error": -0.09069745036936028, | |
| "kl_divergence": 0.006825210554294711 | |
| }, | |
| "choice": { | |
| "count": 3592, | |
| "top1_accuracy": 0.8883630289532294, | |
| "nll": 0.7059347606075163, | |
| "brier_score": 0.026947777171404695, | |
| "ece": 0.18629910839609556, | |
| "signed_calibration_error": -0.18629910839609554, | |
| "kl_divergence": 0.053229628385184406 | |
| }, | |
| "score": { | |
| "count": 3201, | |
| "top1_accuracy": 0.8700406123086536, | |
| "nll": 1.0588078518802615, | |
| "brier_score": 0.010595632707375723, | |
| "ece": 0.31976431260459764, | |
| "signed_calibration_error": -0.3197643126045976, | |
| "kl_divergence": 0.026510583635909378 | |
| } | |
| } | |
| }, | |
| "test": { | |
| "split": "test", | |
| "temperature": 1.0087398771573446, | |
| "default_prediction": "raw", | |
| "calibration_comparison": { | |
| "raw": { | |
| "count": 14261, | |
| "top1_accuracy": 0.8317789776313022, | |
| "nll": 0.6899501951292062, | |
| "brier_score": 0.01078157515435976, | |
| "ece": 0.17475665939191923, | |
| "signed_calibration_error": -0.17475665939191926, | |
| "kl_divergence": 0.021929131222482388, | |
| "confidence_analysis": { | |
| "correct": { | |
| "count": 11862, | |
| "mean": 0.6918767107513397, | |
| "median": 0.6726522445678711, | |
| "std": 0.19655097670181684, | |
| "p25": 0.5013070702552795, | |
| "p75": 0.8612504750490189 | |
| }, | |
| "incorrect": { | |
| "count": 2399, | |
| "mean": 0.4846826750643803, | |
| "median": 0.5003705024719238, | |
| "std": 0.07204632774177197, | |
| "p25": 0.5000030398368835, | |
| "p75": 0.5009814202785492 | |
| } | |
| }, | |
| "confidence_histogram": [ | |
| { | |
| "bucket": "0.0-0.1", | |
| "sample_count": 0, | |
| "average_confidence": null, | |
| "empirical_accuracy": null, | |
| "signed_gap": null | |
| }, | |
| { | |
| "bucket": "0.1-0.2", | |
| "sample_count": 0, | |
| "average_confidence": null, | |
| "empirical_accuracy": null, | |
| "signed_gap": null | |
| }, | |
| { | |
| "bucket": "0.2-0.3", | |
| "sample_count": 147, | |
| "average_confidence": 0.2777280737550891, | |
| "empirical_accuracy": 0.5238095238095238, | |
| "signed_gap": -0.2460814500544347 | |
| }, | |
| { | |
| "bucket": "0.3-0.4", | |
| "sample_count": 683, | |
| "average_confidence": 0.3585107686928041, | |
| "empirical_accuracy": 0.6705710102489019, | |
| "signed_gap": -0.31206024155609774 | |
| }, | |
| { | |
| "bucket": "0.4-0.5", | |
| "sample_count": 1268, | |
| "average_confidence": 0.4527624952915339, | |
| "empirical_accuracy": 0.7626182965299685, | |
| "signed_gap": -0.3098558012384346 | |
| }, | |
| { | |
| "bucket": "0.5-0.6", | |
| "sample_count": 4931, | |
| "average_confidence": 0.5197877843296327, | |
| "empirical_accuracy": 0.6481443926181302, | |
| "signed_gap": -0.1283566082884975 | |
| }, | |
| { | |
| "bucket": "0.6-0.7", | |
| "sample_count": 1700, | |
| "average_confidence": 0.648453773856163, | |
| "empirical_accuracy": 0.971764705882353, | |
| "signed_gap": -0.32331093202619 | |
| }, | |
| { | |
| "bucket": "0.7-0.8", | |
| "sample_count": 1614, | |
| "average_confidence": 0.7490191450054022, | |
| "empirical_accuracy": 0.993184634448575, | |
| "signed_gap": -0.24416548944317273 | |
| }, | |
| { | |
| "bucket": "0.8-0.9", | |
| "sample_count": 1438, | |
| "average_confidence": 0.8470762723510222, | |
| "empirical_accuracy": 0.9986091794158554, | |
| "signed_gap": -0.15153290706483313 | |
| }, | |
| { | |
| "bucket": "0.9-1.0", | |
| "sample_count": 2480, | |
| "average_confidence": 0.9748169333704056, | |
| "empirical_accuracy": 0.9971774193548387, | |
| "signed_gap": -0.022360485984433076 | |
| } | |
| ], | |
| "selective_accuracy": [ | |
| { | |
| "threshold": 0.5, | |
| "coverage": 0.8528854919009887, | |
| "accuracy": 0.8517635451779989, | |
| "risk": 0.1482364548220011, | |
| "sample_count": 12163 | |
| }, | |
| { | |
| "threshold": 0.6, | |
| "coverage": 0.5071173129514059, | |
| "accuracy": 0.9905973451327433, | |
| "risk": 0.009402654867256666, | |
| "sample_count": 7232 | |
| }, | |
| { | |
| "threshold": 0.7, | |
| "coverage": 0.3879110861790898, | |
| "accuracy": 0.9963846710050615, | |
| "risk": 0.003615328994938549, | |
| "sample_count": 5532 | |
| }, | |
| { | |
| "threshold": 0.8, | |
| "coverage": 0.2747352920552556, | |
| "accuracy": 0.9977029096477795, | |
| "risk": 0.0022970903522204766, | |
| "sample_count": 3918 | |
| }, | |
| { | |
| "threshold": 0.9, | |
| "coverage": 0.17390084846784937, | |
| "accuracy": 0.9971774193548387, | |
| "risk": 0.0028225806451612767, | |
| "sample_count": 2480 | |
| }, | |
| { | |
| "threshold": 0.95, | |
| "coverage": 0.1289530888436996, | |
| "accuracy": 0.9978249048395867, | |
| "risk": 0.002175095160413254, | |
| "sample_count": 1839 | |
| } | |
| ] | |
| }, | |
| "temperature_scaled": { | |
| "count": 14261, | |
| "top1_accuracy": 0.8317789776313022, | |
| "nll": 0.6899450200681629, | |
| "brier_score": 0.010779281273617143, | |
| "ece": 0.17597908952728109, | |
| "signed_calibration_error": -0.17597908952728109, | |
| "kl_divergence": 0.02192395616143921, | |
| "confidence_analysis": { | |
| "correct": { | |
| "count": 11862, | |
| "mean": 0.6904957933204784, | |
| "median": 0.6705494859934404, | |
| "std": 0.19655627796667396, | |
| "p25": 0.5012256589733323, | |
| "p75": 0.8592282863558779 | |
| }, | |
| "incorrect": { | |
| "count": 2399, | |
| "mean": 0.484243894907849, | |
| "median": 0.5003640683162293, | |
| "std": 0.07211359411238703, | |
| "p25": 0.500001822502316, | |
| "p75": 0.5009611647486907 | |
| } | |
| }, | |
| "confidence_histogram": [ | |
| { | |
| "bucket": "0.0-0.1", | |
| "sample_count": 0, | |
| "average_confidence": null, | |
| "empirical_accuracy": null, | |
| "signed_gap": null | |
| }, | |
| { | |
| "bucket": "0.1-0.2", | |
| "sample_count": 0, | |
| "average_confidence": null, | |
| "empirical_accuracy": null, | |
| "signed_gap": null | |
| }, | |
| { | |
| "bucket": "0.2-0.3", | |
| "sample_count": 148, | |
| "average_confidence": 0.2771223728628908, | |
| "empirical_accuracy": 0.527027027027027, | |
| "signed_gap": -0.24990465416413615 | |
| }, | |
| { | |
| "bucket": "0.3-0.4", | |
| "sample_count": 689, | |
| "average_confidence": 0.3578369848155047, | |
| "empirical_accuracy": 0.6719883889695211, | |
| "signed_gap": -0.3141514041540164 | |
| }, | |
| { | |
| "bucket": "0.4-0.5", | |
| "sample_count": 1284, | |
| "average_confidence": 0.45233907075338764, | |
| "empirical_accuracy": 0.764018691588785, | |
| "signed_gap": -0.31167962083539735 | |
| }, | |
| { | |
| "bucket": "0.5-0.6", | |
| "sample_count": 4935, | |
| "average_confidence": 0.5197456251441614, | |
| "empirical_accuracy": 0.6488348530901722, | |
| "signed_gap": -0.12908922794601074 | |
| }, | |
| { | |
| "bucket": "0.6-0.7", | |
| "sample_count": 1714, | |
| "average_confidence": 0.6485317314610259, | |
| "empirical_accuracy": 0.9725787631271878, | |
| "signed_gap": -0.3240470316661619 | |
| }, | |
| { | |
| "bucket": "0.7-0.8", | |
| "sample_count": 1609, | |
| "average_confidence": 0.7490936944583456, | |
| "empirical_accuracy": 0.9931634555624611, | |
| "signed_gap": -0.2440697611041155 | |
| }, | |
| { | |
| "bucket": "0.8-0.9", | |
| "sample_count": 1420, | |
| "average_confidence": 0.8467287191781814, | |
| "empirical_accuracy": 0.9985915492957746, | |
| "signed_gap": -0.15186283011759327 | |
| }, | |
| { | |
| "bucket": "0.9-1.0", | |
| "sample_count": 2462, | |
| "average_confidence": 0.9747442974229786, | |
| "empirical_accuracy": 0.9971567831031681, | |
| "signed_gap": -0.022412485680189498 | |
| } | |
| ], | |
| "selective_accuracy": [ | |
| { | |
| "threshold": 0.5, | |
| "coverage": 0.8512727017740691, | |
| "accuracy": 0.8517298187808896, | |
| "risk": 0.1482701812191104, | |
| "sample_count": 12140 | |
| }, | |
| { | |
| "threshold": 0.6, | |
| "coverage": 0.5052240375850221, | |
| "accuracy": 0.9907009021512838, | |
| "risk": 0.009299097848716187, | |
| "sample_count": 7205 | |
| }, | |
| { | |
| "threshold": 0.7, | |
| "coverage": 0.385036112474581, | |
| "accuracy": 0.996357676197414, | |
| "risk": 0.0036423238025860005, | |
| "sample_count": 5491 | |
| }, | |
| { | |
| "threshold": 0.8, | |
| "coverage": 0.27221092490007714, | |
| "accuracy": 0.9976816074188563, | |
| "risk": 0.0023183925811437467, | |
| "sample_count": 3882 | |
| }, | |
| { | |
| "threshold": 0.9, | |
| "coverage": 0.17263866489026014, | |
| "accuracy": 0.9971567831031681, | |
| "risk": 0.0028432168968318683, | |
| "sample_count": 2462 | |
| }, | |
| { | |
| "threshold": 0.95, | |
| "coverage": 0.12811163312530677, | |
| "accuracy": 0.9978106185002736, | |
| "risk": 0.0021893814997263616, | |
| "sample_count": 1827 | |
| } | |
| ] | |
| } | |
| }, | |
| "overall": { | |
| "count": 14261, | |
| "top1_accuracy": 0.8317789776313022, | |
| "nll": 0.6899501951292062, | |
| "brier_score": 0.01078157515435976, | |
| "ece": 0.17475665939191923, | |
| "signed_calibration_error": -0.17475665939191926, | |
| "kl_divergence": 0.021929131222482388, | |
| "confidence_analysis": { | |
| "correct": { | |
| "count": 11862, | |
| "mean": 0.6918767107513397, | |
| "median": 0.6726522445678711, | |
| "std": 0.19655097670181684, | |
| "p25": 0.5013070702552795, | |
| "p75": 0.8612504750490189 | |
| }, | |
| "incorrect": { | |
| "count": 2399, | |
| "mean": 0.4846826750643803, | |
| "median": 0.5003705024719238, | |
| "std": 0.07204632774177197, | |
| "p25": 0.5000030398368835, | |
| "p75": 0.5009814202785492 | |
| } | |
| }, | |
| "confidence_histogram": [ | |
| { | |
| "bucket": "0.0-0.1", | |
| "sample_count": 0, | |
| "average_confidence": null, | |
| "empirical_accuracy": null, | |
| "signed_gap": null | |
| }, | |
| { | |
| "bucket": "0.1-0.2", | |
| "sample_count": 0, | |
| "average_confidence": null, | |
| "empirical_accuracy": null, | |
| "signed_gap": null | |
| }, | |
| { | |
| "bucket": "0.2-0.3", | |
| "sample_count": 147, | |
| "average_confidence": 0.2777280737550891, | |
| "empirical_accuracy": 0.5238095238095238, | |
| "signed_gap": -0.2460814500544347 | |
| }, | |
| { | |
| "bucket": "0.3-0.4", | |
| "sample_count": 683, | |
| "average_confidence": 0.3585107686928041, | |
| "empirical_accuracy": 0.6705710102489019, | |
| "signed_gap": -0.31206024155609774 | |
| }, | |
| { | |
| "bucket": "0.4-0.5", | |
| "sample_count": 1268, | |
| "average_confidence": 0.4527624952915339, | |
| "empirical_accuracy": 0.7626182965299685, | |
| "signed_gap": -0.3098558012384346 | |
| }, | |
| { | |
| "bucket": "0.5-0.6", | |
| "sample_count": 4931, | |
| "average_confidence": 0.5197877843296327, | |
| "empirical_accuracy": 0.6481443926181302, | |
| "signed_gap": -0.1283566082884975 | |
| }, | |
| { | |
| "bucket": "0.6-0.7", | |
| "sample_count": 1700, | |
| "average_confidence": 0.648453773856163, | |
| "empirical_accuracy": 0.971764705882353, | |
| "signed_gap": -0.32331093202619 | |
| }, | |
| { | |
| "bucket": "0.7-0.8", | |
| "sample_count": 1614, | |
| "average_confidence": 0.7490191450054022, | |
| "empirical_accuracy": 0.993184634448575, | |
| "signed_gap": -0.24416548944317273 | |
| }, | |
| { | |
| "bucket": "0.8-0.9", | |
| "sample_count": 1438, | |
| "average_confidence": 0.8470762723510222, | |
| "empirical_accuracy": 0.9986091794158554, | |
| "signed_gap": -0.15153290706483313 | |
| }, | |
| { | |
| "bucket": "0.9-1.0", | |
| "sample_count": 2480, | |
| "average_confidence": 0.9748169333704056, | |
| "empirical_accuracy": 0.9971774193548387, | |
| "signed_gap": -0.022360485984433076 | |
| } | |
| ], | |
| "selective_accuracy": [ | |
| { | |
| "threshold": 0.5, | |
| "coverage": 0.8528854919009887, | |
| "accuracy": 0.8517635451779989, | |
| "risk": 0.1482364548220011, | |
| "sample_count": 12163 | |
| }, | |
| { | |
| "threshold": 0.6, | |
| "coverage": 0.5071173129514059, | |
| "accuracy": 0.9905973451327433, | |
| "risk": 0.009402654867256666, | |
| "sample_count": 7232 | |
| }, | |
| { | |
| "threshold": 0.7, | |
| "coverage": 0.3879110861790898, | |
| "accuracy": 0.9963846710050615, | |
| "risk": 0.003615328994938549, | |
| "sample_count": 5532 | |
| }, | |
| { | |
| "threshold": 0.8, | |
| "coverage": 0.2747352920552556, | |
| "accuracy": 0.9977029096477795, | |
| "risk": 0.0022970903522204766, | |
| "sample_count": 3918 | |
| }, | |
| { | |
| "threshold": 0.9, | |
| "coverage": 0.17390084846784937, | |
| "accuracy": 0.9971774193548387, | |
| "risk": 0.0028225806451612767, | |
| "sample_count": 2480 | |
| }, | |
| { | |
| "threshold": 0.95, | |
| "coverage": 0.1289530888436996, | |
| "accuracy": 0.9978249048395867, | |
| "risk": 0.002175095160413254, | |
| "sample_count": 1839 | |
| } | |
| ] | |
| }, | |
| "by_kind": { | |
| "noul": { | |
| "count": 7298, | |
| "top1_accuracy": 0.785420663195396, | |
| "nll": 0.5104869575155893, | |
| "brier_score": 0.0036984921896671963, | |
| "ece": 0.10110112035656733, | |
| "signed_calibration_error": -0.10110112035656733, | |
| "kl_divergence": 0.005222588741121447 | |
| }, | |
| "choice": { | |
| "count": 3620, | |
| "top1_accuracy": 0.8878453038674033, | |
| "nll": 0.7150685316522385, | |
| "brier_score": 0.025474569714417146, | |
| "ece": 0.19180384800249692, | |
| "signed_calibration_error": -0.1913708825631695, | |
| "kl_divergence": 0.0517853402277982 | |
| }, | |
| "score": { | |
| "count": 3343, | |
| "top1_accuracy": 0.8722704157941968, | |
| "nll": 1.0545312091557688, | |
| "brier_score": 0.010333982922507711, | |
| "ece": 0.3175609178424345, | |
| "signed_calibration_error": -0.3175609178424345, | |
| "kl_divergence": 0.02607058214372944 | |
| } | |
| } | |
| }, | |
| "test_set_30k": { | |
| "split": "test_set_30k", | |
| "temperature": 1.0087398771573446, | |
| "default_prediction": "raw", | |
| "calibration_comparison": { | |
| "raw": { | |
| "count": 29955, | |
| "top1_accuracy": 0.8783174762143214, | |
| "nll": 0.7275528936233122, | |
| "brier_score": 0.011974748942745073, | |
| "ece": 0.21261039866795825, | |
| "signed_calibration_error": -0.21261039866795825, | |
| "kl_divergence": 0.024432378509165548, | |
| "confidence_analysis": { | |
| "correct": { | |
| "count": 26310, | |
| "mean": 0.6923632182076104, | |
| "median": 0.6855365633964539, | |
| "std": 0.18839890771208237, | |
| "p25": 0.5365844368934631, | |
| "p75": 0.8470587730407715 | |
| }, | |
| "incorrect": { | |
| "count": 3645, | |
| "mean": 0.47330020215612706, | |
| "median": 0.5002900958061218, | |
| "std": 0.08985571868253042, | |
| "p25": 0.4233827590942383, | |
| "p75": 0.5083926320075989 | |
| } | |
| }, | |
| "confidence_histogram": [ | |
| { | |
| "bucket": "0.0-0.1", | |
| "sample_count": 0, | |
| "average_confidence": null, | |
| "empirical_accuracy": null, | |
| "signed_gap": null | |
| }, | |
| { | |
| "bucket": "0.1-0.2", | |
| "sample_count": 0, | |
| "average_confidence": null, | |
| "empirical_accuracy": null, | |
| "signed_gap": null | |
| }, | |
| { | |
| "bucket": "0.2-0.3", | |
| "sample_count": 381, | |
| "average_confidence": 0.2765920148512197, | |
| "empirical_accuracy": 0.5380577427821522, | |
| "signed_gap": -0.26146572793093253 | |
| }, | |
| { | |
| "bucket": "0.3-0.4", | |
| "sample_count": 1833, | |
| "average_confidence": 0.3562947420623871, | |
| "empirical_accuracy": 0.6819421713038735, | |
| "signed_gap": -0.3256474292414864 | |
| }, | |
| { | |
| "bucket": "0.4-0.5", | |
| "sample_count": 3110, | |
| "average_confidence": 0.453800807217693, | |
| "empirical_accuracy": 0.7758842443729903, | |
| "signed_gap": -0.3220834371552973 | |
| }, | |
| { | |
| "bucket": "0.5-0.6", | |
| "sample_count": 7576, | |
| "average_confidence": 0.5336691626399724, | |
| "empirical_accuracy": 0.7331045406546991, | |
| "signed_gap": -0.19943537801472666 | |
| }, | |
| { | |
| "bucket": "0.6-0.7", | |
| "sample_count": 4498, | |
| "average_confidence": 0.6493556655979835, | |
| "empirical_accuracy": 0.9717652289906625, | |
| "signed_gap": -0.322409563392679 | |
| }, | |
| { | |
| "bucket": "0.7-0.8", | |
| "sample_count": 4177, | |
| "average_confidence": 0.7498422517198851, | |
| "empirical_accuracy": 0.9944936557337802, | |
| "signed_gap": -0.24465140401389518 | |
| }, | |
| { | |
| "bucket": "0.8-0.9", | |
| "sample_count": 3578, | |
| "average_confidence": 0.8478636906891978, | |
| "empirical_accuracy": 0.9966461710452766, | |
| "signed_gap": -0.14878248035607888 | |
| }, | |
| { | |
| "bucket": "0.9-1.0", | |
| "sample_count": 4802, | |
| "average_confidence": 0.9666469068837037, | |
| "empirical_accuracy": 0.9989587671803415, | |
| "signed_gap": -0.032311860296637795 | |
| } | |
| ], | |
| "selective_accuracy": [ | |
| { | |
| "threshold": 0.5, | |
| "coverage": 0.8222667334334836, | |
| "accuracy": 0.911128253014494, | |
| "risk": 0.08887174698550604, | |
| "sample_count": 24631 | |
| }, | |
| { | |
| "threshold": 0.6, | |
| "coverage": 0.5693540310465699, | |
| "accuracy": 0.9902081501026092, | |
| "risk": 0.009791849897390792, | |
| "sample_count": 17055 | |
| }, | |
| { | |
| "threshold": 0.7, | |
| "coverage": 0.41919545985645135, | |
| "accuracy": 0.9968145257625229, | |
| "risk": 0.0031854742374770595, | |
| "sample_count": 12557 | |
| }, | |
| { | |
| "threshold": 0.8, | |
| "coverage": 0.27975296277749956, | |
| "accuracy": 0.9979713603818616, | |
| "risk": 0.0020286396181383726, | |
| "sample_count": 8380 | |
| }, | |
| { | |
| "threshold": 0.9, | |
| "coverage": 0.16030712735770322, | |
| "accuracy": 0.9989587671803415, | |
| "risk": 0.0010412328196585108, | |
| "sample_count": 4802 | |
| }, | |
| { | |
| "threshold": 0.95, | |
| "coverage": 0.10512435319646136, | |
| "accuracy": 0.9990473166084471, | |
| "risk": 0.0009526833915528554, | |
| "sample_count": 3149 | |
| } | |
| ] | |
| }, | |
| "temperature_scaled": { | |
| "count": 29955, | |
| "top1_accuracy": 0.8783174762143214, | |
| "nll": 0.7275546697338052, | |
| "brier_score": 0.011972585235064483, | |
| "ece": 0.21411426520326776, | |
| "signed_calibration_error": -0.21411426520326776, | |
| "kl_divergence": 0.024434154619658775, | |
| "confidence_analysis": { | |
| "correct": { | |
| "count": 26310, | |
| "mean": 0.6907512556473029, | |
| "median": 0.6833789020182605, | |
| "std": 0.18842788470435384, | |
| "p25": 0.5348970511061539, | |
| "p75": 0.8448785357566411 | |
| }, | |
| "incorrect": { | |
| "count": 3645, | |
| "mean": 0.47257658429508326, | |
| "median": 0.5002725776294035, | |
| "std": 0.08986637932956622, | |
| "p25": 0.4220838770870833, | |
| "p75": 0.5076989910859387 | |
| } | |
| }, | |
| "confidence_histogram": [ | |
| { | |
| "bucket": "0.0-0.1", | |
| "sample_count": 0, | |
| "average_confidence": null, | |
| "empirical_accuracy": null, | |
| "signed_gap": null | |
| }, | |
| { | |
| "bucket": "0.1-0.2", | |
| "sample_count": 0, | |
| "average_confidence": null, | |
| "empirical_accuracy": null, | |
| "signed_gap": null | |
| }, | |
| { | |
| "bucket": "0.2-0.3", | |
| "sample_count": 390, | |
| "average_confidence": 0.27634097301133714, | |
| "empirical_accuracy": 0.5384615384615384, | |
| "signed_gap": -0.2621205654502013 | |
| }, | |
| { | |
| "bucket": "0.3-0.4", | |
| "sample_count": 1863, | |
| "average_confidence": 0.3562694201166133, | |
| "empirical_accuracy": 0.6833064949006978, | |
| "signed_gap": -0.32703707478408456 | |
| }, | |
| { | |
| "bucket": "0.4-0.5", | |
| "sample_count": 3130, | |
| "average_confidence": 0.45376280168734534, | |
| "empirical_accuracy": 0.7757188498402556, | |
| "signed_gap": -0.3219560481529103 | |
| }, | |
| { | |
| "bucket": "0.5-0.6", | |
| "sample_count": 7598, | |
| "average_confidence": 0.5336581786337004, | |
| "empirical_accuracy": 0.735588312713872, | |
| "signed_gap": -0.20193013408017169 | |
| }, | |
| { | |
| "bucket": "0.6-0.7", | |
| "sample_count": 4503, | |
| "average_confidence": 0.6492548349372119, | |
| "empirical_accuracy": 0.9724628025760604, | |
| "signed_gap": -0.3232079676388485 | |
| }, | |
| { | |
| "bucket": "0.7-0.8", | |
| "sample_count": 4183, | |
| "average_confidence": 0.7496742909286859, | |
| "empirical_accuracy": 0.9942624910351422, | |
| "signed_gap": -0.24458820010645632 | |
| }, | |
| { | |
| "bucket": "0.8-0.9", | |
| "sample_count": 3536, | |
| "average_confidence": 0.8476343941651755, | |
| "empirical_accuracy": 0.9968891402714932, | |
| "signed_gap": -0.14925474610631773 | |
| }, | |
| { | |
| "bucket": "0.9-1.0", | |
| "sample_count": 4752, | |
| "average_confidence": 0.966535052125248, | |
| "empirical_accuracy": 0.9989478114478114, | |
| "signed_gap": -0.03241275932256338 | |
| } | |
| ], | |
| "selective_accuracy": [ | |
| { | |
| "threshold": 0.5, | |
| "coverage": 0.8202971123351694, | |
| "accuracy": 0.9115660100927886, | |
| "risk": 0.08843398990721141, | |
| "sample_count": 24572 | |
| }, | |
| { | |
| "threshold": 0.6, | |
| "coverage": 0.5666499749624436, | |
| "accuracy": 0.9903381642512077, | |
| "risk": 0.009661835748792313, | |
| "sample_count": 16974 | |
| }, | |
| { | |
| "threshold": 0.7, | |
| "coverage": 0.41632448673009514, | |
| "accuracy": 0.9967925587362682, | |
| "risk": 0.0032074412637318472, | |
| "sample_count": 12471 | |
| }, | |
| { | |
| "threshold": 0.8, | |
| "coverage": 0.2766816892004674, | |
| "accuracy": 0.9980694980694981, | |
| "risk": 0.0019305019305019266, | |
| "sample_count": 8288 | |
| }, | |
| { | |
| "threshold": 0.9, | |
| "coverage": 0.1586379569354031, | |
| "accuracy": 0.9989478114478114, | |
| "risk": 0.001052188552188582, | |
| "sample_count": 4752 | |
| }, | |
| { | |
| "threshold": 0.95, | |
| "coverage": 0.10392255049240527, | |
| "accuracy": 0.9990362993896563, | |
| "risk": 0.0009637006103436763, | |
| "sample_count": 3113 | |
| } | |
| ] | |
| } | |
| }, | |
| "overall": { | |
| "count": 29955, | |
| "top1_accuracy": 0.8783174762143214, | |
| "nll": 0.7275528936233122, | |
| "brier_score": 0.011974748942745073, | |
| "ece": 0.21261039866795825, | |
| "signed_calibration_error": -0.21261039866795825, | |
| "kl_divergence": 0.024432378509165548, | |
| "confidence_analysis": { | |
| "correct": { | |
| "count": 26310, | |
| "mean": 0.6923632182076104, | |
| "median": 0.6855365633964539, | |
| "std": 0.18839890771208237, | |
| "p25": 0.5365844368934631, | |
| "p75": 0.8470587730407715 | |
| }, | |
| "incorrect": { | |
| "count": 3645, | |
| "mean": 0.47330020215612706, | |
| "median": 0.5002900958061218, | |
| "std": 0.08985571868253042, | |
| "p25": 0.4233827590942383, | |
| "p75": 0.5083926320075989 | |
| } | |
| }, | |
| "confidence_histogram": [ | |
| { | |
| "bucket": "0.0-0.1", | |
| "sample_count": 0, | |
| "average_confidence": null, | |
| "empirical_accuracy": null, | |
| "signed_gap": null | |
| }, | |
| { | |
| "bucket": "0.1-0.2", | |
| "sample_count": 0, | |
| "average_confidence": null, | |
| "empirical_accuracy": null, | |
| "signed_gap": null | |
| }, | |
| { | |
| "bucket": "0.2-0.3", | |
| "sample_count": 381, | |
| "average_confidence": 0.2765920148512197, | |
| "empirical_accuracy": 0.5380577427821522, | |
| "signed_gap": -0.26146572793093253 | |
| }, | |
| { | |
| "bucket": "0.3-0.4", | |
| "sample_count": 1833, | |
| "average_confidence": 0.3562947420623871, | |
| "empirical_accuracy": 0.6819421713038735, | |
| "signed_gap": -0.3256474292414864 | |
| }, | |
| { | |
| "bucket": "0.4-0.5", | |
| "sample_count": 3110, | |
| "average_confidence": 0.453800807217693, | |
| "empirical_accuracy": 0.7758842443729903, | |
| "signed_gap": -0.3220834371552973 | |
| }, | |
| { | |
| "bucket": "0.5-0.6", | |
| "sample_count": 7576, | |
| "average_confidence": 0.5336691626399724, | |
| "empirical_accuracy": 0.7331045406546991, | |
| "signed_gap": -0.19943537801472666 | |
| }, | |
| { | |
| "bucket": "0.6-0.7", | |
| "sample_count": 4498, | |
| "average_confidence": 0.6493556655979835, | |
| "empirical_accuracy": 0.9717652289906625, | |
| "signed_gap": -0.322409563392679 | |
| }, | |
| { | |
| "bucket": "0.7-0.8", | |
| "sample_count": 4177, | |
| "average_confidence": 0.7498422517198851, | |
| "empirical_accuracy": 0.9944936557337802, | |
| "signed_gap": -0.24465140401389518 | |
| }, | |
| { | |
| "bucket": "0.8-0.9", | |
| "sample_count": 3578, | |
| "average_confidence": 0.8478636906891978, | |
| "empirical_accuracy": 0.9966461710452766, | |
| "signed_gap": -0.14878248035607888 | |
| }, | |
| { | |
| "bucket": "0.9-1.0", | |
| "sample_count": 4802, | |
| "average_confidence": 0.9666469068837037, | |
| "empirical_accuracy": 0.9989587671803415, | |
| "signed_gap": -0.032311860296637795 | |
| } | |
| ], | |
| "selective_accuracy": [ | |
| { | |
| "threshold": 0.5, | |
| "coverage": 0.8222667334334836, | |
| "accuracy": 0.911128253014494, | |
| "risk": 0.08887174698550604, | |
| "sample_count": 24631 | |
| }, | |
| { | |
| "threshold": 0.6, | |
| "coverage": 0.5693540310465699, | |
| "accuracy": 0.9902081501026092, | |
| "risk": 0.009791849897390792, | |
| "sample_count": 17055 | |
| }, | |
| { | |
| "threshold": 0.7, | |
| "coverage": 0.41919545985645135, | |
| "accuracy": 0.9968145257625229, | |
| "risk": 0.0031854742374770595, | |
| "sample_count": 12557 | |
| }, | |
| { | |
| "threshold": 0.8, | |
| "coverage": 0.27975296277749956, | |
| "accuracy": 0.9979713603818616, | |
| "risk": 0.0020286396181383726, | |
| "sample_count": 8380 | |
| }, | |
| { | |
| "threshold": 0.9, | |
| "coverage": 0.16030712735770322, | |
| "accuracy": 0.9989587671803415, | |
| "risk": 0.0010412328196585108, | |
| "sample_count": 4802 | |
| }, | |
| { | |
| "threshold": 0.95, | |
| "coverage": 0.10512435319646136, | |
| "accuracy": 0.9990473166084471, | |
| "risk": 0.0009526833915528554, | |
| "sample_count": 3149 | |
| } | |
| ] | |
| }, | |
| "by_kind": { | |
| "noul": { | |
| "count": 12229, | |
| "top1_accuracy": 0.8735791969907597, | |
| "nll": 0.4913208432770474, | |
| "brier_score": 0.003971773635485195, | |
| "ece": 0.14664666192959913, | |
| "signed_calibration_error": -0.14664666192959916, | |
| "kl_divergence": 0.0052616524114012615 | |
| }, | |
| "choice": { | |
| "count": 9199, | |
| "top1_accuracy": 0.8891183824328731, | |
| "nll": 0.7238766215674687, | |
| "brier_score": 0.024050878430150635, | |
| "ece": 0.19473382800328293, | |
| "signed_calibration_error": -0.1947338280032829, | |
| "kl_divergence": 0.048177643130987116 | |
| }, | |
| "score": { | |
| "count": 8527, | |
| "top1_accuracy": 0.8734607716664712, | |
| "nll": 1.0703111638621037, | |
| "brier_score": 0.010424387722836225, | |
| "ece": 0.32649771074931644, | |
| "signed_calibration_error": -0.3264977107493165, | |
| "kl_divergence": 0.026309488887190972 | |
| } | |
| } | |
| }, | |
| "ood": { | |
| "split": "ood", | |
| "temperature": 1.0087398771573446, | |
| "default_prediction": "raw", | |
| "calibration_comparison": { | |
| "raw": { | |
| "count": 13058, | |
| "top1_accuracy": 0.8785418900291009, | |
| "nll": 0.8813702719923984, | |
| "brier_score": 0.19665904522323993, | |
| "ece": 0.0934735002333655, | |
| "signed_calibration_error": 0.09338698955074462, | |
| "kl_divergence": 0.8662859112694271, | |
| "confidence_analysis": { | |
| "correct": { | |
| "count": 11472, | |
| "mean": 0.9861596955215062, | |
| "median": 0.9999983310699463, | |
| "std": 0.07204877546218105, | |
| "p25": 0.9999236762523651, | |
| "p75": 0.9999998807907104 | |
| }, | |
| "incorrect": { | |
| "count": 1586, | |
| "mean": 0.8689932424532811, | |
| "median": 0.9988584518432617, | |
| "std": 0.21735096923047184, | |
| "p25": 0.8063253164291382, | |
| "p75": 0.9999670684337616 | |
| } | |
| }, | |
| "confidence_histogram": [ | |
| { | |
| "bucket": "0.0-0.1", | |
| "sample_count": 0, | |
| "average_confidence": null, | |
| "empirical_accuracy": null, | |
| "signed_gap": null | |
| }, | |
| { | |
| "bucket": "0.1-0.2", | |
| "sample_count": 1, | |
| "average_confidence": 0.19825825095176697, | |
| "empirical_accuracy": 0.0, | |
| "signed_gap": 0.19825825095176697 | |
| }, | |
| { | |
| "bucket": "0.2-0.3", | |
| "sample_count": 65, | |
| "average_confidence": 0.256245233462407, | |
| "empirical_accuracy": 0.23076923076923078, | |
| "signed_gap": 0.025476002693176236 | |
| }, | |
| { | |
| "bucket": "0.3-0.4", | |
| "sample_count": 44, | |
| "average_confidence": 0.34126521985639224, | |
| "empirical_accuracy": 0.25, | |
| "signed_gap": 0.09126521985639224 | |
| }, | |
| { | |
| "bucket": "0.4-0.5", | |
| "sample_count": 25, | |
| "average_confidence": 0.4627038073539734, | |
| "empirical_accuracy": 0.52, | |
| "signed_gap": -0.057296192646026645 | |
| }, | |
| { | |
| "bucket": "0.5-0.6", | |
| "sample_count": 357, | |
| "average_confidence": 0.5382261586790326, | |
| "empirical_accuracy": 0.39775910364145656, | |
| "signed_gap": 0.140467055037576 | |
| }, | |
| { | |
| "bucket": "0.6-0.7", | |
| "sample_count": 103, | |
| "average_confidence": 0.6438586167918826, | |
| "empirical_accuracy": 0.5825242718446602, | |
| "signed_gap": 0.06133434494722245 | |
| }, | |
| { | |
| "bucket": "0.7-0.8", | |
| "sample_count": 100, | |
| "average_confidence": 0.7468274587392807, | |
| "empirical_accuracy": 0.61, | |
| "signed_gap": 0.1368274587392807 | |
| }, | |
| { | |
| "bucket": "0.8-0.9", | |
| "sample_count": 172, | |
| "average_confidence": 0.8525181381508361, | |
| "empirical_accuracy": 0.6046511627906976, | |
| "signed_gap": 0.24786697536013846 | |
| }, | |
| { | |
| "bucket": "0.9-1.0", | |
| "sample_count": 12191, | |
| "average_confidence": 0.998132212653288, | |
| "empirical_accuracy": 0.9077188089574276, | |
| "signed_gap": 0.09041340369586037 | |
| } | |
| ], | |
| "selective_accuracy": [ | |
| { | |
| "threshold": 0.5, | |
| "coverage": 0.989661510185327, | |
| "accuracy": 0.8847016946529443, | |
| "risk": 0.11529830534705565, | |
| "sample_count": 12923 | |
| }, | |
| { | |
| "threshold": 0.6, | |
| "coverage": 0.9623219482309695, | |
| "accuracy": 0.8985357313385326, | |
| "risk": 0.1014642686614674, | |
| "sample_count": 12566 | |
| }, | |
| { | |
| "threshold": 0.7, | |
| "coverage": 0.9544340634094042, | |
| "accuracy": 0.9011473962930273, | |
| "risk": 0.09885260370697269, | |
| "sample_count": 12463 | |
| }, | |
| { | |
| "threshold": 0.8, | |
| "coverage": 0.9467759228059427, | |
| "accuracy": 0.9035023861522284, | |
| "risk": 0.09649761384777156, | |
| "sample_count": 12363 | |
| }, | |
| { | |
| "threshold": 0.9, | |
| "coverage": 0.9336039209679889, | |
| "accuracy": 0.9077188089574276, | |
| "risk": 0.09228119104257237, | |
| "sample_count": 12191 | |
| }, | |
| { | |
| "threshold": 0.95, | |
| "coverage": 0.9241078266196967, | |
| "accuracy": 0.9104168393138311, | |
| "risk": 0.0895831606861689, | |
| "sample_count": 12067 | |
| } | |
| ] | |
| }, | |
| "temperature_scaled": { | |
| "count": 13058, | |
| "top1_accuracy": 0.8785418900291009, | |
| "nll": 0.8743518421149571, | |
| "brier_score": 0.19656952511113734, | |
| "ece": 0.09334952718772531, | |
| "signed_calibration_error": 0.09325994317919446, | |
| "kl_divergence": 0.8592674813919858, | |
| "confidence_analysis": { | |
| "correct": { | |
| "count": 11472, | |
| "mean": 0.9860722430341967, | |
| "median": 0.9999981217543632, | |
| "std": 0.07221466062590726, | |
| "p25": 0.999917101407375, | |
| "p75": 0.9999998629109141 | |
| }, | |
| "incorrect": { | |
| "count": 1586, | |
| "mean": 0.8685798019833653, | |
| "median": 0.9987841752508422, | |
| "std": 0.2175917495077864, | |
| "p25": 0.8042375494799926, | |
| "p75": 0.9999636629198553 | |
| } | |
| }, | |
| "confidence_histogram": [ | |
| { | |
| "bucket": "0.0-0.1", | |
| "sample_count": 0, | |
| "average_confidence": null, | |
| "empirical_accuracy": null, | |
| "signed_gap": null | |
| }, | |
| { | |
| "bucket": "0.1-0.2", | |
| "sample_count": 1, | |
| "average_confidence": 0.19798991716161005, | |
| "empirical_accuracy": 0.0, | |
| "signed_gap": 0.19798991716161005 | |
| }, | |
| { | |
| "bucket": "0.2-0.3", | |
| "sample_count": 67, | |
| "average_confidence": 0.25693683657772004, | |
| "empirical_accuracy": 0.22388059701492538, | |
| "signed_gap": 0.03305623956279466 | |
| }, | |
| { | |
| "bucket": "0.3-0.4", | |
| "sample_count": 42, | |
| "average_confidence": 0.3418979937441187, | |
| "empirical_accuracy": 0.2619047619047619, | |
| "signed_gap": 0.0799932318393568 | |
| }, | |
| { | |
| "bucket": "0.4-0.5", | |
| "sample_count": 25, | |
| "average_confidence": 0.4615572793674953, | |
| "empirical_accuracy": 0.52, | |
| "signed_gap": -0.05844272063250472 | |
| }, | |
| { | |
| "bucket": "0.5-0.6", | |
| "sample_count": 358, | |
| "average_confidence": 0.5379884025898151, | |
| "empirical_accuracy": 0.3994413407821229, | |
| "signed_gap": 0.13854706180769216 | |
| }, | |
| { | |
| "bucket": "0.6-0.7", | |
| "sample_count": 103, | |
| "average_confidence": 0.6434854397989195, | |
| "empirical_accuracy": 0.5825242718446602, | |
| "signed_gap": 0.06096116795425932 | |
| }, | |
| { | |
| "bucket": "0.7-0.8", | |
| "sample_count": 103, | |
| "average_confidence": 0.7474879529885827, | |
| "empirical_accuracy": 0.6019417475728155, | |
| "signed_gap": 0.14554620541576724 | |
| }, | |
| { | |
| "bucket": "0.8-0.9", | |
| "sample_count": 177, | |
| "average_confidence": 0.8541454633210329, | |
| "empirical_accuracy": 0.6045197740112994, | |
| "signed_gap": 0.2496256893097335 | |
| }, | |
| { | |
| "bucket": "0.9-1.0", | |
| "sample_count": 12182, | |
| "average_confidence": 0.9981466982229948, | |
| "empirical_accuracy": 0.9079789853882778, | |
| "signed_gap": 0.09016771283471703 | |
| } | |
| ], | |
| "selective_accuracy": [ | |
| { | |
| "threshold": 0.5, | |
| "coverage": 0.989661510185327, | |
| "accuracy": 0.8847016946529443, | |
| "risk": 0.11529830534705565, | |
| "sample_count": 12923 | |
| }, | |
| { | |
| "threshold": 0.6, | |
| "coverage": 0.9622453668249349, | |
| "accuracy": 0.8985276561878233, | |
| "risk": 0.10147234381217674, | |
| "sample_count": 12565 | |
| }, | |
| { | |
| "threshold": 0.7, | |
| "coverage": 0.9543574820033696, | |
| "accuracy": 0.9011394639704702, | |
| "risk": 0.09886053602952982, | |
| "sample_count": 12462 | |
| }, | |
| { | |
| "threshold": 0.8, | |
| "coverage": 0.9464695971818042, | |
| "accuracy": 0.9036329800145643, | |
| "risk": 0.09636701998543573, | |
| "sample_count": 12359 | |
| }, | |
| { | |
| "threshold": 0.9, | |
| "coverage": 0.9329146883136774, | |
| "accuracy": 0.9079789853882778, | |
| "risk": 0.09202101461172218, | |
| "sample_count": 12182 | |
| }, | |
| { | |
| "threshold": 0.95, | |
| "coverage": 0.9235717567774544, | |
| "accuracy": 0.9104477611940298, | |
| "risk": 0.08955223880597019, | |
| "sample_count": 12060 | |
| } | |
| ] | |
| } | |
| }, | |
| "overall": { | |
| "count": 13058, | |
| "top1_accuracy": 0.8785418900291009, | |
| "nll": 0.8813702719923984, | |
| "brier_score": 0.19665904522323993, | |
| "ece": 0.0934735002333655, | |
| "signed_calibration_error": 0.09338698955074462, | |
| "kl_divergence": 0.8662859112694271, | |
| "confidence_analysis": { | |
| "correct": { | |
| "count": 11472, | |
| "mean": 0.9861596955215062, | |
| "median": 0.9999983310699463, | |
| "std": 0.07204877546218105, | |
| "p25": 0.9999236762523651, | |
| "p75": 0.9999998807907104 | |
| }, | |
| "incorrect": { | |
| "count": 1586, | |
| "mean": 0.8689932424532811, | |
| "median": 0.9988584518432617, | |
| "std": 0.21735096923047184, | |
| "p25": 0.8063253164291382, | |
| "p75": 0.9999670684337616 | |
| } | |
| }, | |
| "confidence_histogram": [ | |
| { | |
| "bucket": "0.0-0.1", | |
| "sample_count": 0, | |
| "average_confidence": null, | |
| "empirical_accuracy": null, | |
| "signed_gap": null | |
| }, | |
| { | |
| "bucket": "0.1-0.2", | |
| "sample_count": 1, | |
| "average_confidence": 0.19825825095176697, | |
| "empirical_accuracy": 0.0, | |
| "signed_gap": 0.19825825095176697 | |
| }, | |
| { | |
| "bucket": "0.2-0.3", | |
| "sample_count": 65, | |
| "average_confidence": 0.256245233462407, | |
| "empirical_accuracy": 0.23076923076923078, | |
| "signed_gap": 0.025476002693176236 | |
| }, | |
| { | |
| "bucket": "0.3-0.4", | |
| "sample_count": 44, | |
| "average_confidence": 0.34126521985639224, | |
| "empirical_accuracy": 0.25, | |
| "signed_gap": 0.09126521985639224 | |
| }, | |
| { | |
| "bucket": "0.4-0.5", | |
| "sample_count": 25, | |
| "average_confidence": 0.4627038073539734, | |
| "empirical_accuracy": 0.52, | |
| "signed_gap": -0.057296192646026645 | |
| }, | |
| { | |
| "bucket": "0.5-0.6", | |
| "sample_count": 357, | |
| "average_confidence": 0.5382261586790326, | |
| "empirical_accuracy": 0.39775910364145656, | |
| "signed_gap": 0.140467055037576 | |
| }, | |
| { | |
| "bucket": "0.6-0.7", | |
| "sample_count": 103, | |
| "average_confidence": 0.6438586167918826, | |
| "empirical_accuracy": 0.5825242718446602, | |
| "signed_gap": 0.06133434494722245 | |
| }, | |
| { | |
| "bucket": "0.7-0.8", | |
| "sample_count": 100, | |
| "average_confidence": 0.7468274587392807, | |
| "empirical_accuracy": 0.61, | |
| "signed_gap": 0.1368274587392807 | |
| }, | |
| { | |
| "bucket": "0.8-0.9", | |
| "sample_count": 172, | |
| "average_confidence": 0.8525181381508361, | |
| "empirical_accuracy": 0.6046511627906976, | |
| "signed_gap": 0.24786697536013846 | |
| }, | |
| { | |
| "bucket": "0.9-1.0", | |
| "sample_count": 12191, | |
| "average_confidence": 0.998132212653288, | |
| "empirical_accuracy": 0.9077188089574276, | |
| "signed_gap": 0.09041340369586037 | |
| } | |
| ], | |
| "selective_accuracy": [ | |
| { | |
| "threshold": 0.5, | |
| "coverage": 0.989661510185327, | |
| "accuracy": 0.8847016946529443, | |
| "risk": 0.11529830534705565, | |
| "sample_count": 12923 | |
| }, | |
| { | |
| "threshold": 0.6, | |
| "coverage": 0.9623219482309695, | |
| "accuracy": 0.8985357313385326, | |
| "risk": 0.1014642686614674, | |
| "sample_count": 12566 | |
| }, | |
| { | |
| "threshold": 0.7, | |
| "coverage": 0.9544340634094042, | |
| "accuracy": 0.9011473962930273, | |
| "risk": 0.09885260370697269, | |
| "sample_count": 12463 | |
| }, | |
| { | |
| "threshold": 0.8, | |
| "coverage": 0.9467759228059427, | |
| "accuracy": 0.9035023861522284, | |
| "risk": 0.09649761384777156, | |
| "sample_count": 12363 | |
| }, | |
| { | |
| "threshold": 0.9, | |
| "coverage": 0.9336039209679889, | |
| "accuracy": 0.9077188089574276, | |
| "risk": 0.09228119104257237, | |
| "sample_count": 12191 | |
| }, | |
| { | |
| "threshold": 0.95, | |
| "coverage": 0.9241078266196967, | |
| "accuracy": 0.9104168393138311, | |
| "risk": 0.0895831606861689, | |
| "sample_count": 12067 | |
| } | |
| ] | |
| }, | |
| "by_kind": { | |
| "noul": { | |
| "count": 9767, | |
| "top1_accuracy": 0.9117436264973892, | |
| "nll": 0.6845514018050354, | |
| "brier_score": 0.1665514502307479, | |
| "ece": 0.07861264344307467, | |
| "signed_calibration_error": 0.07850355333421202, | |
| "kl_divergence": 0.6845514018050354 | |
| }, | |
| "choice": { | |
| "count": 3219, | |
| "top1_accuracy": 0.791860826343585, | |
| "nll": 1.4607612426226197, | |
| "brier_score": 0.2742619765350296, | |
| "ece": 0.13922969467819907, | |
| "signed_calibration_error": 0.13852634314944115, | |
| "kl_divergence": 1.399570940565907 | |
| }, | |
| "score": { | |
| "count": 72, | |
| "top1_accuracy": 0.25, | |
| "nll": 1.6767920867325716, | |
| "brier_score": 0.8113402174318239, | |
| "ece": 0.1267397738993168, | |
| "signed_calibration_error": 0.09426118806004524, | |
| "kl_divergence": 1.6767920867325716 | |
| } | |
| } | |
| } | |
| } |