{
  "calibration": {
    "split": "calibration",
    "temperature": 1.0087398771573446,
    "default_prediction": "raw",
    "calibration_comparison": {
      "raw": {
        "count": 13766,
        "top1_accuracy": 0.8311782652912973,
        "nll": 0.6855602746553805,
        "brier_score": 0.010973403472421008,
        "ece": 0.1712850628941104,
        "signed_calibration_error": -0.17128506289411038,
        "kl_divergence": 0.02185337568727864,
        "confidence_analysis": {
          "correct": {
            "count": 11442,
            "mean": 0.694725726679517,
            "median": 0.6793243885040283,
            "std": 0.19659272893691354,
            "p25": 0.501576766371727,
            "p75": 0.8654205501079559
          },
          "incorrect": {
            "count": 2324,
            "mean": 0.48839847656309915,
            "median": 0.500387966632843,
            "std": 0.07211813579079812,
            "p25": 0.5000417530536652,
            "p75": 0.5010467320680618
          }
        },
        "confidence_histogram": [
          {
            "bucket": "0.0-0.1",
            "sample_count": 0,
            "average_confidence": null,
            "empirical_accuracy": null,
            "signed_gap": null
          },
          {
            "bucket": "0.1-0.2",
            "sample_count": 0,
            "average_confidence": null,
            "empirical_accuracy": null,
            "signed_gap": null
          },
          {
            "bucket": "0.2-0.3",
            "sample_count": 134,
            "average_confidence": 0.27603658633445627,
            "empirical_accuracy": 0.5,
            "signed_gap": -0.22396341366554373
          },
          {
            "bucket": "0.3-0.4",
            "sample_count": 637,
            "average_confidence": 0.3581763524649843,
            "empirical_accuracy": 0.6923076923076923,
            "signed_gap": -0.334131339842708
          },
          {
            "bucket": "0.4-0.5",
            "sample_count": 1197,
            "average_confidence": 0.45263352763583087,
            "empirical_accuracy": 0.7861319966583125,
            "signed_gap": -0.3334984690224816
          },
          {
            "bucket": "0.5-0.6",
            "sample_count": 4710,
            "average_confidence": 0.520006962819717,
            "empirical_accuracy": 0.6322717622080679,
            "signed_gap": -0.11226479938835088
          },
          {
            "bucket": "0.6-0.7",
            "sample_count": 1701,
            "average_confidence": 0.6498155774683338,
            "empirical_accuracy": 0.9711934156378601,
            "signed_gap": -0.32137783816952625
          },
          {
            "bucket": "0.7-0.8",
            "sample_count": 1530,
            "average_confidence": 0.7490799561045528,
            "empirical_accuracy": 0.9888888888888889,
            "signed_gap": -0.23980893278433613
          },
          {
            "bucket": "0.8-0.9",
            "sample_count": 1452,
            "average_confidence": 0.847257420872197,
            "empirical_accuracy": 0.9986225895316805,
            "signed_gap": -0.15136516865948346
          },
          {
            "bucket": "0.9-1.0",
            "sample_count": 2405,
            "average_confidence": 0.9755763211765804,
            "empirical_accuracy": 0.997920997920998,
            "signed_gap": -0.022344676744417535
          }
        ],
        "selective_accuracy": [
          {
            "threshold": 0.5,
            "coverage": 0.8570390817957286,
            "accuracy": 0.8470079674521105,
            "risk": 0.1529920325478895,
            "sample_count": 11798
          },
          {
            "threshold": 0.6,
            "coverage": 0.514891762312945,
            "accuracy": 0.9897009029345373,
            "risk": 0.010299097065462726,
            "sample_count": 7088
          },
          {
            "threshold": 0.7,
            "coverage": 0.39132645648699693,
            "accuracy": 0.9955448301466493,
            "risk": 0.0044551698533507045,
            "sample_count": 5387
          },
          {
            "threshold": 0.8,
            "coverage": 0.28018305971233476,
            "accuracy": 0.9981851179673321,
            "risk": 0.001814882032667886,
            "sample_count": 3857
          },
          {
            "threshold": 0.9,
            "coverage": 0.1747057968908906,
            "accuracy": 0.997920997920998,
            "risk": 0.0020790020790020236,
            "sample_count": 2405
          },
          {
            "threshold": 0.95,
            "coverage": 0.13162865029783524,
            "accuracy": 0.9977924944812362,
            "risk": 0.002207505518763808,
            "sample_count": 1812
          }
        ]
      },
      "temperature_scaled": {
        "count": 13766,
        "top1_accuracy": 0.8311782652912973,
        "nll": 0.6855494345867709,
        "brier_score": 0.010966401509070522,
        "ece": 0.17250904779846635,
        "signed_calibration_error": -0.17250904779846632,
        "kl_divergence": 0.02184253561866892,
        "confidence_analysis": {
          "correct": {
            "count": 11442,
            "mean": 0.6933398573451267,
            "median": 0.6774639946027864,
            "std": 0.19661340445165715,
            "p25": 0.5014006484541963,
            "p75": 0.863254437984726
          },
          "incorrect": {
            "count": 2324,
            "mean": 0.4879715147432757,
            "median": 0.5003803821956974,
            "std": 0.07215117413704505,
            "p25": 0.5000357963828128,
            "p75": 0.5010135416360336
          }
        },
        "confidence_histogram": [
          {
            "bucket": "0.0-0.1",
            "sample_count": 0,
            "average_confidence": null,
            "empirical_accuracy": null,
            "signed_gap": null
          },
          {
            "bucket": "0.1-0.2",
            "sample_count": 0,
            "average_confidence": null,
            "empirical_accuracy": null,
            "signed_gap": null
          },
          {
            "bucket": "0.2-0.3",
            "sample_count": 137,
            "average_confidence": 0.2758391267365239,
            "empirical_accuracy": 0.5036496350364964,
            "signed_gap": -0.22781050829997251
          },
          {
            "bucket": "0.3-0.4",
            "sample_count": 642,
            "average_confidence": 0.3577477496059755,
            "empirical_accuracy": 0.6962616822429907,
            "signed_gap": -0.3385139326370152
          },
          {
            "bucket": "0.4-0.5",
            "sample_count": 1215,
            "average_confidence": 0.4524140225007448,
            "empirical_accuracy": 0.7851851851851852,
            "signed_gap": -0.3327711626844404
          },
          {
            "bucket": "0.5-0.6",
            "sample_count": 4721,
            "average_confidence": 0.5201900803979108,
            "empirical_accuracy": 0.633552213514086,
            "signed_gap": -0.11336213311617516
          },
          {
            "bucket": "0.6-0.7",
            "sample_count": 1701,
            "average_confidence": 0.6500267694102276,
            "empirical_accuracy": 0.9723691945914168,
            "signed_gap": -0.32234242518118916
          },
          {
            "bucket": "0.7-0.8",
            "sample_count": 1529,
            "average_confidence": 0.7491738888998721,
            "empirical_accuracy": 0.9895356442119032,
            "signed_gap": -0.24036175531203108
          },
          {
            "bucket": "0.8-0.9",
            "sample_count": 1445,
            "average_confidence": 0.8472853994689213,
            "empirical_accuracy": 0.9986159169550173,
            "signed_gap": -0.15133051748609594
          },
          {
            "bucket": "0.9-1.0",
            "sample_count": 2376,
            "average_confidence": 0.9759117055438289,
            "empirical_accuracy": 0.997895622895623,
            "signed_gap": -0.021983917351794058
          }
        ],
        "selective_accuracy": [
          {
            "threshold": 0.5,
            "coverage": 0.8551503704779893,
            "accuracy": 0.8470948012232415,
            "risk": 0.15290519877675846,
            "sample_count": 11772
          },
          {
            "threshold": 0.6,
            "coverage": 0.5122039808223159,
            "accuracy": 0.9900723301659339,
            "risk": 0.009927669834066122,
            "sample_count": 7051
          },
          {
            "threshold": 0.7,
            "coverage": 0.38863867499636784,
            "accuracy": 0.9957009345794392,
            "risk": 0.004299065420560799,
            "sample_count": 5350
          },
          {
            "threshold": 0.8,
            "coverage": 0.2775679209646956,
            "accuracy": 0.9981680188432348,
            "risk": 0.0018319811567651945,
            "sample_count": 3821
          },
          {
            "threshold": 0.9,
            "coverage": 0.17259915734418133,
            "accuracy": 0.997895622895623,
            "risk": 0.002104377104377053,
            "sample_count": 2376
          },
          {
            "threshold": 0.95,
            "coverage": 0.1306116518959756,
            "accuracy": 0.9977753058954394,
            "risk": 0.0022246941045606095,
            "sample_count": 1798
          }
        ]
      }
    },
    "overall": {
      "count": 13766,
      "top1_accuracy": 0.8311782652912973,
      "nll": 0.6855602746553805,
      "brier_score": 0.010973403472421008,
      "ece": 0.1712850628941104,
      "signed_calibration_error": -0.17128506289411038,
      "kl_divergence": 0.02185337568727864,
      "confidence_analysis": {
        "correct": {
          "count": 11442,
          "mean": 0.694725726679517,
          "median": 0.6793243885040283,
          "std": 0.19659272893691354,
          "p25": 0.501576766371727,
          "p75": 0.8654205501079559
        },
        "incorrect": {
          "count": 2324,
          "mean": 0.48839847656309915,
          "median": 0.500387966632843,
          "std": 0.07211813579079812,
          "p25": 0.5000417530536652,
          "p75": 0.5010467320680618
        }
      },
      "confidence_histogram": [
        {
          "bucket": "0.0-0.1",
          "sample_count": 0,
          "average_confidence": null,
          "empirical_accuracy": null,
          "signed_gap": null
        },
        {
          "bucket": "0.1-0.2",
          "sample_count": 0,
          "average_confidence": null,
          "empirical_accuracy": null,
          "signed_gap": null
        },
        {
          "bucket": "0.2-0.3",
          "sample_count": 134,
          "average_confidence": 0.27603658633445627,
          "empirical_accuracy": 0.5,
          "signed_gap": -0.22396341366554373
        },
        {
          "bucket": "0.3-0.4",
          "sample_count": 637,
          "average_confidence": 0.3581763524649843,
          "empirical_accuracy": 0.6923076923076923,
          "signed_gap": -0.334131339842708
        },
        {
          "bucket": "0.4-0.5",
          "sample_count": 1197,
          "average_confidence": 0.45263352763583087,
          "empirical_accuracy": 0.7861319966583125,
          "signed_gap": -0.3334984690224816
        },
        {
          "bucket": "0.5-0.6",
          "sample_count": 4710,
          "average_confidence": 0.520006962819717,
          "empirical_accuracy": 0.6322717622080679,
          "signed_gap": -0.11226479938835088
        },
        {
          "bucket": "0.6-0.7",
          "sample_count": 1701,
          "average_confidence": 0.6498155774683338,
          "empirical_accuracy": 0.9711934156378601,
          "signed_gap": -0.32137783816952625
        },
        {
          "bucket": "0.7-0.8",
          "sample_count": 1530,
          "average_confidence": 0.7490799561045528,
          "empirical_accuracy": 0.9888888888888889,
          "signed_gap": -0.23980893278433613
        },
        {
          "bucket": "0.8-0.9",
          "sample_count": 1452,
          "average_confidence": 0.847257420872197,
          "empirical_accuracy": 0.9986225895316805,
          "signed_gap": -0.15136516865948346
        },
        {
          "bucket": "0.9-1.0",
          "sample_count": 2405,
          "average_confidence": 0.9755763211765804,
          "empirical_accuracy": 0.997920997920998,
          "signed_gap": -0.022344676744417535
        }
      ],
      "selective_accuracy": [
        {
          "threshold": 0.5,
          "coverage": 0.8570390817957286,
          "accuracy": 0.8470079674521105,
          "risk": 0.1529920325478895,
          "sample_count": 11798
        },
        {
          "threshold": 0.6,
          "coverage": 0.514891762312945,
          "accuracy": 0.9897009029345373,
          "risk": 0.010299097065462726,
          "sample_count": 7088
        },
        {
          "threshold": 0.7,
          "coverage": 0.39132645648699693,
          "accuracy": 0.9955448301466493,
          "risk": 0.0044551698533507045,
          "sample_count": 5387
        },
        {
          "threshold": 0.8,
          "coverage": 0.28018305971233476,
          "accuracy": 0.9981851179673321,
          "risk": 0.001814882032667886,
          "sample_count": 3857
        },
        {
          "threshold": 0.9,
          "coverage": 0.1747057968908906,
          "accuracy": 0.997920997920998,
          "risk": 0.0020790020790020236,
          "sample_count": 2405
        },
        {
          "threshold": 0.95,
          "coverage": 0.13162865029783524,
          "accuracy": 0.9977924944812362,
          "risk": 0.002207505518763808,
          "sample_count": 1812
        }
      ]
    },
    "by_kind": {
      "noul": {
        "count": 7175,
        "top1_accuracy": 0.7839721254355401,
        "nll": 0.5094520074091426,
        "brier_score": 0.0037153609942311735,
        "ece": 0.0968418979893994,
        "signed_calibration_error": -0.09684189798939934,
        "kl_divergence": 0.005356521734346802
      },
      "choice": {
        "count": 3418,
        "top1_accuracy": 0.8838502047981276,
        "nll": 0.7117832643875154,
        "brier_score": 0.0267761779649997,
        "ece": 0.18598052990966682,
        "signed_calibration_error": -0.18598052990966682,
        "kl_divergence": 0.05244051993410699
      },
      "score": {
        "count": 3173,
        "top1_accuracy": 0.8811849984242042,
        "nll": 1.055540305726077,
        "brier_score": 0.010362805163369038,
        "ece": 0.323790767883751,
        "signed_calibration_error": -0.323790767883751,
        "kl_divergence": 0.026208266351201326
      }
    }
  },
  "validation": {
    "split": "validation",
    "temperature": 1.0087398771573446,
    "default_prediction": "raw",
    "calibration_comparison": {
      "raw": {
        "count": 14111,
        "top1_accuracy": 0.8263765856424067,
        "nll": 0.6840500261181026,
        "brier_score": 0.011582265527708373,
        "ece": 0.16699567031458232,
        "signed_calibration_error": -0.1669956703145823,
        "kl_divergence": 0.02310311772478613,
        "confidence_analysis": {
          "correct": {
            "count": 11661,
            "mean": 0.6960411451404903,
            "median": 0.6765217781066895,
            "std": 0.19690960502309285,
            "p25": 0.50345379114151,
            "p75": 0.8713327646255493
          },
          "incorrect": {
            "count": 2450,
            "mean": 0.4848931847786417,
            "median": 0.500392496585846,
            "std": 0.07416498496851862,
            "p25": 0.5000284314155579,
            "p75": 0.5010085254907608
          }
        },
        "confidence_histogram": [
          {
            "bucket": "0.0-0.1",
            "sample_count": 0,
            "average_confidence": null,
            "empirical_accuracy": null,
            "signed_gap": null
          },
          {
            "bucket": "0.1-0.2",
            "sample_count": 0,
            "average_confidence": null,
            "empirical_accuracy": null,
            "signed_gap": null
          },
          {
            "bucket": "0.2-0.3",
            "sample_count": 148,
            "average_confidence": 0.27538005906987834,
            "empirical_accuracy": 0.46621621621621623,
            "signed_gap": -0.1908361571463379
          },
          {
            "bucket": "0.3-0.4",
            "sample_count": 667,
            "average_confidence": 0.3564207344755776,
            "empirical_accuracy": 0.664167916041979,
            "signed_gap": -0.3077471815664014
          },
          {
            "bucket": "0.4-0.5",
            "sample_count": 1160,
            "average_confidence": 0.4550788116095395,
            "empirical_accuracy": 0.7741379310344828,
            "signed_gap": -0.3190591194249433
          },
          {
            "bucket": "0.5-0.6",
            "sample_count": 4972,
            "average_confidence": 0.5200908452703985,
            "empirical_accuracy": 0.6351568785197104,
            "signed_gap": -0.11506603324931186
          },
          {
            "bucket": "0.6-0.7",
            "sample_count": 1678,
            "average_confidence": 0.6489259644685684,
            "empirical_accuracy": 0.9713945172824792,
            "signed_gap": -0.32246855281391074
          },
          {
            "bucket": "0.7-0.8",
            "sample_count": 1540,
            "average_confidence": 0.7498124199641215,
            "empirical_accuracy": 0.9935064935064936,
            "signed_gap": -0.24369407354237205
          },
          {
            "bucket": "0.8-0.9",
            "sample_count": 1412,
            "average_confidence": 0.8490157142154894,
            "empirical_accuracy": 0.9936260623229461,
            "signed_gap": -0.14461034810745677
          },
          {
            "bucket": "0.9-1.0",
            "sample_count": 2534,
            "average_confidence": 0.9746775966087663,
            "empirical_accuracy": 0.9984214680347278,
            "signed_gap": -0.02374387142596146
          }
        ],
        "selective_accuracy": [
          {
            "threshold": 0.5,
            "coverage": 0.8600382680178584,
            "accuracy": 0.844676994067238,
            "risk": 0.15532300593276205,
            "sample_count": 12136
          },
          {
            "threshold": 0.6,
            "coverage": 0.5076890369215505,
            "accuracy": 0.9900893355667225,
            "risk": 0.009910664433277483,
            "sample_count": 7164
          },
          {
            "threshold": 0.7,
            "coverage": 0.3887747147615336,
            "accuracy": 0.9958075100255195,
            "risk": 0.004192489974480518,
            "sample_count": 5486
          },
          {
            "threshold": 0.8,
            "coverage": 0.279639997165332,
            "accuracy": 0.996705524581855,
            "risk": 0.003294475418144982,
            "sample_count": 3946
          },
          {
            "threshold": 0.9,
            "coverage": 0.17957621713556798,
            "accuracy": 0.9984214680347278,
            "risk": 0.0015785319652722452,
            "sample_count": 2534
          },
          {
            "threshold": 0.95,
            "coverage": 0.13273332860888667,
            "accuracy": 0.9983982915109451,
            "risk": 0.0016017084890549427,
            "sample_count": 1873
          }
        ]
      },
      "temperature_scaled": {
        "count": 14111,
        "top1_accuracy": 0.8263057189426688,
        "nll": 0.6840468913421749,
        "brier_score": 0.011582336771387319,
        "ece": 0.16813608304214336,
        "signed_calibration_error": -0.16813608304214334,
        "kl_divergence": 0.023099982948858303,
        "confidence_analysis": {
          "correct": {
            "count": 11660,
            "mean": 0.694681208951386,
            "median": 0.6747364841480057,
            "std": 0.19693845421787254,
            "p25": 0.5021719095333224,
            "p75": 0.8689303683324948
          },
          "incorrect": {
            "count": 2451,
            "mean": 0.4844752492122222,
            "median": 0.5003890293824186,
            "std": 0.07420726284342961,
            "p25": 0.500027128879432,
            "p75": 0.5009866390613054
          }
        },
        "confidence_histogram": [
          {
            "bucket": "0.0-0.1",
            "sample_count": 0,
            "average_confidence": null,
            "empirical_accuracy": null,
            "signed_gap": null
          },
          {
            "bucket": "0.1-0.2",
            "sample_count": 0,
            "average_confidence": null,
            "empirical_accuracy": null,
            "signed_gap": null
          },
          {
            "bucket": "0.2-0.3",
            "sample_count": 151,
            "average_confidence": 0.2751785209954829,
            "empirical_accuracy": 0.46357615894039733,
            "signed_gap": -0.18839763794491443
          },
          {
            "bucket": "0.3-0.4",
            "sample_count": 674,
            "average_confidence": 0.35612814661984515,
            "empirical_accuracy": 0.6646884272997032,
            "signed_gap": -0.3085602806798581
          },
          {
            "bucket": "0.4-0.5",
            "sample_count": 1168,
            "average_confidence": 0.45463179276212934,
            "empirical_accuracy": 0.7782534246575342,
            "signed_gap": -0.3236216318954049
          },
          {
            "bucket": "0.5-0.6",
            "sample_count": 4987,
            "average_confidence": 0.5201110232818913,
            "empirical_accuracy": 0.6360537397232805,
            "signed_gap": -0.1159427164413892
          },
          {
            "bucket": "0.6-0.7",
            "sample_count": 1673,
            "average_confidence": 0.6487813857971743,
            "empirical_accuracy": 0.9719067543335326,
            "signed_gap": -0.3231253685363583
          },
          {
            "bucket": "0.7-0.8",
            "sample_count": 1546,
            "average_confidence": 0.7495579579669318,
            "empirical_accuracy": 0.9935316946959897,
            "signed_gap": -0.24397373672905787
          },
          {
            "bucket": "0.8-0.9",
            "sample_count": 1409,
            "average_confidence": 0.8491268025105129,
            "empirical_accuracy": 0.99361249112846,
            "signed_gap": -0.14448568861794708
          },
          {
            "bucket": "0.9-1.0",
            "sample_count": 2503,
            "average_confidence": 0.9749893213960854,
            "empirical_accuracy": 0.9984019176987615,
            "signed_gap": -0.023412596302676114
          }
        ],
        "selective_accuracy": [
          {
            "threshold": 0.5,
            "coverage": 0.8587626674225781,
            "accuracy": 0.8444462782637399,
            "risk": 0.15555372173626014,
            "sample_count": 12118
          },
          {
            "threshold": 0.6,
            "coverage": 0.5053504358302033,
            "accuracy": 0.9901837049502173,
            "risk": 0.009816295049782653,
            "sample_count": 7131
          },
          {
            "threshold": 0.7,
            "coverage": 0.38679044716887534,
            "accuracy": 0.9957860021986076,
            "risk": 0.004213997801392422,
            "sample_count": 5458
          },
          {
            "threshold": 0.8,
            "coverage": 0.277230529374247,
            "accuracy": 0.996676891615542,
            "risk": 0.00332310838445804,
            "sample_count": 3912
          },
          {
            "threshold": 0.9,
            "coverage": 0.1773793494436964,
            "accuracy": 0.9984019176987615,
            "risk": 0.0015980823012384793,
            "sample_count": 2503
          },
          {
            "threshold": 0.95,
            "coverage": 0.131599461413082,
            "accuracy": 0.9983844911147012,
            "risk": 0.001615508885298822,
            "sample_count": 1857
          }
        ]
      }
    },
    "overall": {
      "count": 14111,
      "top1_accuracy": 0.8263765856424067,
      "nll": 0.6840500261181026,
      "brier_score": 0.011582265527708373,
      "ece": 0.16699567031458232,
      "signed_calibration_error": -0.1669956703145823,
      "kl_divergence": 0.02310311772478613,
      "confidence_analysis": {
        "correct": {
          "count": 11661,
          "mean": 0.6960411451404903,
          "median": 0.6765217781066895,
          "std": 0.19690960502309285,
          "p25": 0.50345379114151,
          "p75": 0.8713327646255493
        },
        "incorrect": {
          "count": 2450,
          "mean": 0.4848931847786417,
          "median": 0.500392496585846,
          "std": 0.07416498496851862,
          "p25": 0.5000284314155579,
          "p75": 0.5010085254907608
        }
      },
      "confidence_histogram": [
        {
          "bucket": "0.0-0.1",
          "sample_count": 0,
          "average_confidence": null,
          "empirical_accuracy": null,
          "signed_gap": null
        },
        {
          "bucket": "0.1-0.2",
          "sample_count": 0,
          "average_confidence": null,
          "empirical_accuracy": null,
          "signed_gap": null
        },
        {
          "bucket": "0.2-0.3",
          "sample_count": 148,
          "average_confidence": 0.27538005906987834,
          "empirical_accuracy": 0.46621621621621623,
          "signed_gap": -0.1908361571463379
        },
        {
          "bucket": "0.3-0.4",
          "sample_count": 667,
          "average_confidence": 0.3564207344755776,
          "empirical_accuracy": 0.664167916041979,
          "signed_gap": -0.3077471815664014
        },
        {
          "bucket": "0.4-0.5",
          "sample_count": 1160,
          "average_confidence": 0.4550788116095395,
          "empirical_accuracy": 0.7741379310344828,
          "signed_gap": -0.3190591194249433
        },
        {
          "bucket": "0.5-0.6",
          "sample_count": 4972,
          "average_confidence": 0.5200908452703985,
          "empirical_accuracy": 0.6351568785197104,
          "signed_gap": -0.11506603324931186
        },
        {
          "bucket": "0.6-0.7",
          "sample_count": 1678,
          "average_confidence": 0.6489259644685684,
          "empirical_accuracy": 0.9713945172824792,
          "signed_gap": -0.32246855281391074
        },
        {
          "bucket": "0.7-0.8",
          "sample_count": 1540,
          "average_confidence": 0.7498124199641215,
          "empirical_accuracy": 0.9935064935064936,
          "signed_gap": -0.24369407354237205
        },
        {
          "bucket": "0.8-0.9",
          "sample_count": 1412,
          "average_confidence": 0.8490157142154894,
          "empirical_accuracy": 0.9936260623229461,
          "signed_gap": -0.14461034810745677
        },
        {
          "bucket": "0.9-1.0",
          "sample_count": 2534,
          "average_confidence": 0.9746775966087663,
          "empirical_accuracy": 0.9984214680347278,
          "signed_gap": -0.02374387142596146
        }
      ],
      "selective_accuracy": [
        {
          "threshold": 0.5,
          "coverage": 0.8600382680178584,
          "accuracy": 0.844676994067238,
          "risk": 0.15532300593276205,
          "sample_count": 12136
        },
        {
          "threshold": 0.6,
          "coverage": 0.5076890369215505,
          "accuracy": 0.9900893355667225,
          "risk": 0.009910664433277483,
          "sample_count": 7164
        },
        {
          "threshold": 0.7,
          "coverage": 0.3887747147615336,
          "accuracy": 0.9958075100255195,
          "risk": 0.004192489974480518,
          "sample_count": 5486
        },
        {
          "threshold": 0.8,
          "coverage": 0.279639997165332,
          "accuracy": 0.996705524581855,
          "risk": 0.003294475418144982,
          "sample_count": 3946
        },
        {
          "threshold": 0.9,
          "coverage": 0.17957621713556798,
          "accuracy": 0.9984214680347278,
          "risk": 0.0015785319652722452,
          "sample_count": 2534
        },
        {
          "threshold": 0.95,
          "coverage": 0.13273332860888667,
          "accuracy": 0.9983982915109451,
          "risk": 0.0016017084890549427,
          "sample_count": 1873
        }
      ]
    },
    "by_kind": {
      "noul": {
        "count": 7318,
        "top1_accuracy": 0.7768515987974857,
        "nll": 0.5093834824517124,
        "brier_score": 0.004471756349480391,
        "ece": 0.0906974503693603,
        "signed_calibration_error": -0.09069745036936028,
        "kl_divergence": 0.006825210554294711
      },
      "choice": {
        "count": 3592,
        "top1_accuracy": 0.8883630289532294,
        "nll": 0.7059347606075163,
        "brier_score": 0.026947777171404695,
        "ece": 0.18629910839609556,
        "signed_calibration_error": -0.18629910839609554,
        "kl_divergence": 0.053229628385184406
      },
      "score": {
        "count": 3201,
        "top1_accuracy": 0.8700406123086536,
        "nll": 1.0588078518802615,
        "brier_score": 0.010595632707375723,
        "ece": 0.31976431260459764,
        "signed_calibration_error": -0.3197643126045976,
        "kl_divergence": 0.026510583635909378
      }
    }
  },
  "test": {
    "split": "test",
    "temperature": 1.0087398771573446,
    "default_prediction": "raw",
    "calibration_comparison": {
      "raw": {
        "count": 14261,
        "top1_accuracy": 0.8317789776313022,
        "nll": 0.6899501951292062,
        "brier_score": 0.01078157515435976,
        "ece": 0.17475665939191923,
        "signed_calibration_error": -0.17475665939191926,
        "kl_divergence": 0.021929131222482388,
        "confidence_analysis": {
          "correct": {
            "count": 11862,
            "mean": 0.6918767107513397,
            "median": 0.6726522445678711,
            "std": 0.19655097670181684,
            "p25": 0.5013070702552795,
            "p75": 0.8612504750490189
          },
          "incorrect": {
            "count": 2399,
            "mean": 0.4846826750643803,
            "median": 0.5003705024719238,
            "std": 0.07204632774177197,
            "p25": 0.5000030398368835,
            "p75": 0.5009814202785492
          }
        },
        "confidence_histogram": [
          {
            "bucket": "0.0-0.1",
            "sample_count": 0,
            "average_confidence": null,
            "empirical_accuracy": null,
            "signed_gap": null
          },
          {
            "bucket": "0.1-0.2",
            "sample_count": 0,
            "average_confidence": null,
            "empirical_accuracy": null,
            "signed_gap": null
          },
          {
            "bucket": "0.2-0.3",
            "sample_count": 147,
            "average_confidence": 0.2777280737550891,
            "empirical_accuracy": 0.5238095238095238,
            "signed_gap": -0.2460814500544347
          },
          {
            "bucket": "0.3-0.4",
            "sample_count": 683,
            "average_confidence": 0.3585107686928041,
            "empirical_accuracy": 0.6705710102489019,
            "signed_gap": -0.31206024155609774
          },
          {
            "bucket": "0.4-0.5",
            "sample_count": 1268,
            "average_confidence": 0.4527624952915339,
            "empirical_accuracy": 0.7626182965299685,
            "signed_gap": -0.3098558012384346
          },
          {
            "bucket": "0.5-0.6",
            "sample_count": 4931,
            "average_confidence": 0.5197877843296327,
            "empirical_accuracy": 0.6481443926181302,
            "signed_gap": -0.1283566082884975
          },
          {
            "bucket": "0.6-0.7",
            "sample_count": 1700,
            "average_confidence": 0.648453773856163,
            "empirical_accuracy": 0.971764705882353,
            "signed_gap": -0.32331093202619
          },
          {
            "bucket": "0.7-0.8",
            "sample_count": 1614,
            "average_confidence": 0.7490191450054022,
            "empirical_accuracy": 0.993184634448575,
            "signed_gap": -0.24416548944317273
          },
          {
            "bucket": "0.8-0.9",
            "sample_count": 1438,
            "average_confidence": 0.8470762723510222,
            "empirical_accuracy": 0.9986091794158554,
            "signed_gap": -0.15153290706483313
          },
          {
            "bucket": "0.9-1.0",
            "sample_count": 2480,
            "average_confidence": 0.9748169333704056,
            "empirical_accuracy": 0.9971774193548387,
            "signed_gap": -0.022360485984433076
          }
        ],
        "selective_accuracy": [
          {
            "threshold": 0.5,
            "coverage": 0.8528854919009887,
            "accuracy": 0.8517635451779989,
            "risk": 0.1482364548220011,
            "sample_count": 12163
          },
          {
            "threshold": 0.6,
            "coverage": 0.5071173129514059,
            "accuracy": 0.9905973451327433,
            "risk": 0.009402654867256666,
            "sample_count": 7232
          },
          {
            "threshold": 0.7,
            "coverage": 0.3879110861790898,
            "accuracy": 0.9963846710050615,
            "risk": 0.003615328994938549,
            "sample_count": 5532
          },
          {
            "threshold": 0.8,
            "coverage": 0.2747352920552556,
            "accuracy": 0.9977029096477795,
            "risk": 0.0022970903522204766,
            "sample_count": 3918
          },
          {
            "threshold": 0.9,
            "coverage": 0.17390084846784937,
            "accuracy": 0.9971774193548387,
            "risk": 0.0028225806451612767,
            "sample_count": 2480
          },
          {
            "threshold": 0.95,
            "coverage": 0.1289530888436996,
            "accuracy": 0.9978249048395867,
            "risk": 0.002175095160413254,
            "sample_count": 1839
          }
        ]
      },
      "temperature_scaled": {
        "count": 14261,
        "top1_accuracy": 0.8317789776313022,
        "nll": 0.6899450200681629,
        "brier_score": 0.010779281273617143,
        "ece": 0.17597908952728109,
        "signed_calibration_error": -0.17597908952728109,
        "kl_divergence": 0.02192395616143921,
        "confidence_analysis": {
          "correct": {
            "count": 11862,
            "mean": 0.6904957933204784,
            "median": 0.6705494859934404,
            "std": 0.19655627796667396,
            "p25": 0.5012256589733323,
            "p75": 0.8592282863558779
          },
          "incorrect": {
            "count": 2399,
            "mean": 0.484243894907849,
            "median": 0.5003640683162293,
            "std": 0.07211359411238703,
            "p25": 0.500001822502316,
            "p75": 0.5009611647486907
          }
        },
        "confidence_histogram": [
          {
            "bucket": "0.0-0.1",
            "sample_count": 0,
            "average_confidence": null,
            "empirical_accuracy": null,
            "signed_gap": null
          },
          {
            "bucket": "0.1-0.2",
            "sample_count": 0,
            "average_confidence": null,
            "empirical_accuracy": null,
            "signed_gap": null
          },
          {
            "bucket": "0.2-0.3",
            "sample_count": 148,
            "average_confidence": 0.2771223728628908,
            "empirical_accuracy": 0.527027027027027,
            "signed_gap": -0.24990465416413615
          },
          {
            "bucket": "0.3-0.4",
            "sample_count": 689,
            "average_confidence": 0.3578369848155047,
            "empirical_accuracy": 0.6719883889695211,
            "signed_gap": -0.3141514041540164
          },
          {
            "bucket": "0.4-0.5",
            "sample_count": 1284,
            "average_confidence": 0.45233907075338764,
            "empirical_accuracy": 0.764018691588785,
            "signed_gap": -0.31167962083539735
          },
          {
            "bucket": "0.5-0.6",
            "sample_count": 4935,
            "average_confidence": 0.5197456251441614,
            "empirical_accuracy": 0.6488348530901722,
            "signed_gap": -0.12908922794601074
          },
          {
            "bucket": "0.6-0.7",
            "sample_count": 1714,
            "average_confidence": 0.6485317314610259,
            "empirical_accuracy": 0.9725787631271878,
            "signed_gap": -0.3240470316661619
          },
          {
            "bucket": "0.7-0.8",
            "sample_count": 1609,
            "average_confidence": 0.7490936944583456,
            "empirical_accuracy": 0.9931634555624611,
            "signed_gap": -0.2440697611041155
          },
          {
            "bucket": "0.8-0.9",
            "sample_count": 1420,
            "average_confidence": 0.8467287191781814,
            "empirical_accuracy": 0.9985915492957746,
            "signed_gap": -0.15186283011759327
          },
          {
            "bucket": "0.9-1.0",
            "sample_count": 2462,
            "average_confidence": 0.9747442974229786,
            "empirical_accuracy": 0.9971567831031681,
            "signed_gap": -0.022412485680189498
          }
        ],
        "selective_accuracy": [
          {
            "threshold": 0.5,
            "coverage": 0.8512727017740691,
            "accuracy": 0.8517298187808896,
            "risk": 0.1482701812191104,
            "sample_count": 12140
          },
          {
            "threshold": 0.6,
            "coverage": 0.5052240375850221,
            "accuracy": 0.9907009021512838,
            "risk": 0.009299097848716187,
            "sample_count": 7205
          },
          {
            "threshold": 0.7,
            "coverage": 0.385036112474581,
            "accuracy": 0.996357676197414,
            "risk": 0.0036423238025860005,
            "sample_count": 5491
          },
          {
            "threshold": 0.8,
            "coverage": 0.27221092490007714,
            "accuracy": 0.9976816074188563,
            "risk": 0.0023183925811437467,
            "sample_count": 3882
          },
          {
            "threshold": 0.9,
            "coverage": 0.17263866489026014,
            "accuracy": 0.9971567831031681,
            "risk": 0.0028432168968318683,
            "sample_count": 2462
          },
          {
            "threshold": 0.95,
            "coverage": 0.12811163312530677,
            "accuracy": 0.9978106185002736,
            "risk": 0.0021893814997263616,
            "sample_count": 1827
          }
        ]
      }
    },
    "overall": {
      "count": 14261,
      "top1_accuracy": 0.8317789776313022,
      "nll": 0.6899501951292062,
      "brier_score": 0.01078157515435976,
      "ece": 0.17475665939191923,
      "signed_calibration_error": -0.17475665939191926,
      "kl_divergence": 0.021929131222482388,
      "confidence_analysis": {
        "correct": {
          "count": 11862,
          "mean": 0.6918767107513397,
          "median": 0.6726522445678711,
          "std": 0.19655097670181684,
          "p25": 0.5013070702552795,
          "p75": 0.8612504750490189
        },
        "incorrect": {
          "count": 2399,
          "mean": 0.4846826750643803,
          "median": 0.5003705024719238,
          "std": 0.07204632774177197,
          "p25": 0.5000030398368835,
          "p75": 0.5009814202785492
        }
      },
      "confidence_histogram": [
        {
          "bucket": "0.0-0.1",
          "sample_count": 0,
          "average_confidence": null,
          "empirical_accuracy": null,
          "signed_gap": null
        },
        {
          "bucket": "0.1-0.2",
          "sample_count": 0,
          "average_confidence": null,
          "empirical_accuracy": null,
          "signed_gap": null
        },
        {
          "bucket": "0.2-0.3",
          "sample_count": 147,
          "average_confidence": 0.2777280737550891,
          "empirical_accuracy": 0.5238095238095238,
          "signed_gap": -0.2460814500544347
        },
        {
          "bucket": "0.3-0.4",
          "sample_count": 683,
          "average_confidence": 0.3585107686928041,
          "empirical_accuracy": 0.6705710102489019,
          "signed_gap": -0.31206024155609774
        },
        {
          "bucket": "0.4-0.5",
          "sample_count": 1268,
          "average_confidence": 0.4527624952915339,
          "empirical_accuracy": 0.7626182965299685,
          "signed_gap": -0.3098558012384346
        },
        {
          "bucket": "0.5-0.6",
          "sample_count": 4931,
          "average_confidence": 0.5197877843296327,
          "empirical_accuracy": 0.6481443926181302,
          "signed_gap": -0.1283566082884975
        },
        {
          "bucket": "0.6-0.7",
          "sample_count": 1700,
          "average_confidence": 0.648453773856163,
          "empirical_accuracy": 0.971764705882353,
          "signed_gap": -0.32331093202619
        },
        {
          "bucket": "0.7-0.8",
          "sample_count": 1614,
          "average_confidence": 0.7490191450054022,
          "empirical_accuracy": 0.993184634448575,
          "signed_gap": -0.24416548944317273
        },
        {
          "bucket": "0.8-0.9",
          "sample_count": 1438,
          "average_confidence": 0.8470762723510222,
          "empirical_accuracy": 0.9986091794158554,
          "signed_gap": -0.15153290706483313
        },
        {
          "bucket": "0.9-1.0",
          "sample_count": 2480,
          "average_confidence": 0.9748169333704056,
          "empirical_accuracy": 0.9971774193548387,
          "signed_gap": -0.022360485984433076
        }
      ],
      "selective_accuracy": [
        {
          "threshold": 0.5,
          "coverage": 0.8528854919009887,
          "accuracy": 0.8517635451779989,
          "risk": 0.1482364548220011,
          "sample_count": 12163
        },
        {
          "threshold": 0.6,
          "coverage": 0.5071173129514059,
          "accuracy": 0.9905973451327433,
          "risk": 0.009402654867256666,
          "sample_count": 7232
        },
        {
          "threshold": 0.7,
          "coverage": 0.3879110861790898,
          "accuracy": 0.9963846710050615,
          "risk": 0.003615328994938549,
          "sample_count": 5532
        },
        {
          "threshold": 0.8,
          "coverage": 0.2747352920552556,
          "accuracy": 0.9977029096477795,
          "risk": 0.0022970903522204766,
          "sample_count": 3918
        },
        {
          "threshold": 0.9,
          "coverage": 0.17390084846784937,
          "accuracy": 0.9971774193548387,
          "risk": 0.0028225806451612767,
          "sample_count": 2480
        },
        {
          "threshold": 0.95,
          "coverage": 0.1289530888436996,
          "accuracy": 0.9978249048395867,
          "risk": 0.002175095160413254,
          "sample_count": 1839
        }
      ]
    },
    "by_kind": {
      "noul": {
        "count": 7298,
        "top1_accuracy": 0.785420663195396,
        "nll": 0.5104869575155893,
        "brier_score": 0.0036984921896671963,
        "ece": 0.10110112035656733,
        "signed_calibration_error": -0.10110112035656733,
        "kl_divergence": 0.005222588741121447
      },
      "choice": {
        "count": 3620,
        "top1_accuracy": 0.8878453038674033,
        "nll": 0.7150685316522385,
        "brier_score": 0.025474569714417146,
        "ece": 0.19180384800249692,
        "signed_calibration_error": -0.1913708825631695,
        "kl_divergence": 0.0517853402277982
      },
      "score": {
        "count": 3343,
        "top1_accuracy": 0.8722704157941968,
        "nll": 1.0545312091557688,
        "brier_score": 0.010333982922507711,
        "ece": 0.3175609178424345,
        "signed_calibration_error": -0.3175609178424345,
        "kl_divergence": 0.02607058214372944
      }
    }
  },
  "test_set_30k": {
    "split": "test_set_30k",
    "temperature": 1.0087398771573446,
    "default_prediction": "raw",
    "calibration_comparison": {
      "raw": {
        "count": 29955,
        "top1_accuracy": 0.8783174762143214,
        "nll": 0.7275528936233122,
        "brier_score": 0.011974748942745073,
        "ece": 0.21261039866795825,
        "signed_calibration_error": -0.21261039866795825,
        "kl_divergence": 0.024432378509165548,
        "confidence_analysis": {
          "correct": {
            "count": 26310,
            "mean": 0.6923632182076104,
            "median": 0.6855365633964539,
            "std": 0.18839890771208237,
            "p25": 0.5365844368934631,
            "p75": 0.8470587730407715
          },
          "incorrect": {
            "count": 3645,
            "mean": 0.47330020215612706,
            "median": 0.5002900958061218,
            "std": 0.08985571868253042,
            "p25": 0.4233827590942383,
            "p75": 0.5083926320075989
          }
        },
        "confidence_histogram": [
          {
            "bucket": "0.0-0.1",
            "sample_count": 0,
            "average_confidence": null,
            "empirical_accuracy": null,
            "signed_gap": null
          },
          {
            "bucket": "0.1-0.2",
            "sample_count": 0,
            "average_confidence": null,
            "empirical_accuracy": null,
            "signed_gap": null
          },
          {
            "bucket": "0.2-0.3",
            "sample_count": 381,
            "average_confidence": 0.2765920148512197,
            "empirical_accuracy": 0.5380577427821522,
            "signed_gap": -0.26146572793093253
          },
          {
            "bucket": "0.3-0.4",
            "sample_count": 1833,
            "average_confidence": 0.3562947420623871,
            "empirical_accuracy": 0.6819421713038735,
            "signed_gap": -0.3256474292414864
          },
          {
            "bucket": "0.4-0.5",
            "sample_count": 3110,
            "average_confidence": 0.453800807217693,
            "empirical_accuracy": 0.7758842443729903,
            "signed_gap": -0.3220834371552973
          },
          {
            "bucket": "0.5-0.6",
            "sample_count": 7576,
            "average_confidence": 0.5336691626399724,
            "empirical_accuracy": 0.7331045406546991,
            "signed_gap": -0.19943537801472666
          },
          {
            "bucket": "0.6-0.7",
            "sample_count": 4498,
            "average_confidence": 0.6493556655979835,
            "empirical_accuracy": 0.9717652289906625,
            "signed_gap": -0.322409563392679
          },
          {
            "bucket": "0.7-0.8",
            "sample_count": 4177,
            "average_confidence": 0.7498422517198851,
            "empirical_accuracy": 0.9944936557337802,
            "signed_gap": -0.24465140401389518
          },
          {
            "bucket": "0.8-0.9",
            "sample_count": 3578,
            "average_confidence": 0.8478636906891978,
            "empirical_accuracy": 0.9966461710452766,
            "signed_gap": -0.14878248035607888
          },
          {
            "bucket": "0.9-1.0",
            "sample_count": 4802,
            "average_confidence": 0.9666469068837037,
            "empirical_accuracy": 0.9989587671803415,
            "signed_gap": -0.032311860296637795
          }
        ],
        "selective_accuracy": [
          {
            "threshold": 0.5,
            "coverage": 0.8222667334334836,
            "accuracy": 0.911128253014494,
            "risk": 0.08887174698550604,
            "sample_count": 24631
          },
          {
            "threshold": 0.6,
            "coverage": 0.5693540310465699,
            "accuracy": 0.9902081501026092,
            "risk": 0.009791849897390792,
            "sample_count": 17055
          },
          {
            "threshold": 0.7,
            "coverage": 0.41919545985645135,
            "accuracy": 0.9968145257625229,
            "risk": 0.0031854742374770595,
            "sample_count": 12557
          },
          {
            "threshold": 0.8,
            "coverage": 0.27975296277749956,
            "accuracy": 0.9979713603818616,
            "risk": 0.0020286396181383726,
            "sample_count": 8380
          },
          {
            "threshold": 0.9,
            "coverage": 0.16030712735770322,
            "accuracy": 0.9989587671803415,
            "risk": 0.0010412328196585108,
            "sample_count": 4802
          },
          {
            "threshold": 0.95,
            "coverage": 0.10512435319646136,
            "accuracy": 0.9990473166084471,
            "risk": 0.0009526833915528554,
            "sample_count": 3149
          }
        ]
      },
      "temperature_scaled": {
        "count": 29955,
        "top1_accuracy": 0.8783174762143214,
        "nll": 0.7275546697338052,
        "brier_score": 0.011972585235064483,
        "ece": 0.21411426520326776,
        "signed_calibration_error": -0.21411426520326776,
        "kl_divergence": 0.024434154619658775,
        "confidence_analysis": {
          "correct": {
            "count": 26310,
            "mean": 0.6907512556473029,
            "median": 0.6833789020182605,
            "std": 0.18842788470435384,
            "p25": 0.5348970511061539,
            "p75": 0.8448785357566411
          },
          "incorrect": {
            "count": 3645,
            "mean": 0.47257658429508326,
            "median": 0.5002725776294035,
            "std": 0.08986637932956622,
            "p25": 0.4220838770870833,
            "p75": 0.5076989910859387
          }
        },
        "confidence_histogram": [
          {
            "bucket": "0.0-0.1",
            "sample_count": 0,
            "average_confidence": null,
            "empirical_accuracy": null,
            "signed_gap": null
          },
          {
            "bucket": "0.1-0.2",
            "sample_count": 0,
            "average_confidence": null,
            "empirical_accuracy": null,
            "signed_gap": null
          },
          {
            "bucket": "0.2-0.3",
            "sample_count": 390,
            "average_confidence": 0.27634097301133714,
            "empirical_accuracy": 0.5384615384615384,
            "signed_gap": -0.2621205654502013
          },
          {
            "bucket": "0.3-0.4",
            "sample_count": 1863,
            "average_confidence": 0.3562694201166133,
            "empirical_accuracy": 0.6833064949006978,
            "signed_gap": -0.32703707478408456
          },
          {
            "bucket": "0.4-0.5",
            "sample_count": 3130,
            "average_confidence": 0.45376280168734534,
            "empirical_accuracy": 0.7757188498402556,
            "signed_gap": -0.3219560481529103
          },
          {
            "bucket": "0.5-0.6",
            "sample_count": 7598,
            "average_confidence": 0.5336581786337004,
            "empirical_accuracy": 0.735588312713872,
            "signed_gap": -0.20193013408017169
          },
          {
            "bucket": "0.6-0.7",
            "sample_count": 4503,
            "average_confidence": 0.6492548349372119,
            "empirical_accuracy": 0.9724628025760604,
            "signed_gap": -0.3232079676388485
          },
          {
            "bucket": "0.7-0.8",
            "sample_count": 4183,
            "average_confidence": 0.7496742909286859,
            "empirical_accuracy": 0.9942624910351422,
            "signed_gap": -0.24458820010645632
          },
          {
            "bucket": "0.8-0.9",
            "sample_count": 3536,
            "average_confidence": 0.8476343941651755,
            "empirical_accuracy": 0.9968891402714932,
            "signed_gap": -0.14925474610631773
          },
          {
            "bucket": "0.9-1.0",
            "sample_count": 4752,
            "average_confidence": 0.966535052125248,
            "empirical_accuracy": 0.9989478114478114,
            "signed_gap": -0.03241275932256338
          }
        ],
        "selective_accuracy": [
          {
            "threshold": 0.5,
            "coverage": 0.8202971123351694,
            "accuracy": 0.9115660100927886,
            "risk": 0.08843398990721141,
            "sample_count": 24572
          },
          {
            "threshold": 0.6,
            "coverage": 0.5666499749624436,
            "accuracy": 0.9903381642512077,
            "risk": 0.009661835748792313,
            "sample_count": 16974
          },
          {
            "threshold": 0.7,
            "coverage": 0.41632448673009514,
            "accuracy": 0.9967925587362682,
            "risk": 0.0032074412637318472,
            "sample_count": 12471
          },
          {
            "threshold": 0.8,
            "coverage": 0.2766816892004674,
            "accuracy": 0.9980694980694981,
            "risk": 0.0019305019305019266,
            "sample_count": 8288
          },
          {
            "threshold": 0.9,
            "coverage": 0.1586379569354031,
            "accuracy": 0.9989478114478114,
            "risk": 0.001052188552188582,
            "sample_count": 4752
          },
          {
            "threshold": 0.95,
            "coverage": 0.10392255049240527,
            "accuracy": 0.9990362993896563,
            "risk": 0.0009637006103436763,
            "sample_count": 3113
          }
        ]
      }
    },
    "overall": {
      "count": 29955,
      "top1_accuracy": 0.8783174762143214,
      "nll": 0.7275528936233122,
      "brier_score": 0.011974748942745073,
      "ece": 0.21261039866795825,
      "signed_calibration_error": -0.21261039866795825,
      "kl_divergence": 0.024432378509165548,
      "confidence_analysis": {
        "correct": {
          "count": 26310,
          "mean": 0.6923632182076104,
          "median": 0.6855365633964539,
          "std": 0.18839890771208237,
          "p25": 0.5365844368934631,
          "p75": 0.8470587730407715
        },
        "incorrect": {
          "count": 3645,
          "mean": 0.47330020215612706,
          "median": 0.5002900958061218,
          "std": 0.08985571868253042,
          "p25": 0.4233827590942383,
          "p75": 0.5083926320075989
        }
      },
      "confidence_histogram": [
        {
          "bucket": "0.0-0.1",
          "sample_count": 0,
          "average_confidence": null,
          "empirical_accuracy": null,
          "signed_gap": null
        },
        {
          "bucket": "0.1-0.2",
          "sample_count": 0,
          "average_confidence": null,
          "empirical_accuracy": null,
          "signed_gap": null
        },
        {
          "bucket": "0.2-0.3",
          "sample_count": 381,
          "average_confidence": 0.2765920148512197,
          "empirical_accuracy": 0.5380577427821522,
          "signed_gap": -0.26146572793093253
        },
        {
          "bucket": "0.3-0.4",
          "sample_count": 1833,
          "average_confidence": 0.3562947420623871,
          "empirical_accuracy": 0.6819421713038735,
          "signed_gap": -0.3256474292414864
        },
        {
          "bucket": "0.4-0.5",
          "sample_count": 3110,
          "average_confidence": 0.453800807217693,
          "empirical_accuracy": 0.7758842443729903,
          "signed_gap": -0.3220834371552973
        },
        {
          "bucket": "0.5-0.6",
          "sample_count": 7576,
          "average_confidence": 0.5336691626399724,
          "empirical_accuracy": 0.7331045406546991,
          "signed_gap": -0.19943537801472666
        },
        {
          "bucket": "0.6-0.7",
          "sample_count": 4498,
          "average_confidence": 0.6493556655979835,
          "empirical_accuracy": 0.9717652289906625,
          "signed_gap": -0.322409563392679
        },
        {
          "bucket": "0.7-0.8",
          "sample_count": 4177,
          "average_confidence": 0.7498422517198851,
          "empirical_accuracy": 0.9944936557337802,
          "signed_gap": -0.24465140401389518
        },
        {
          "bucket": "0.8-0.9",
          "sample_count": 3578,
          "average_confidence": 0.8478636906891978,
          "empirical_accuracy": 0.9966461710452766,
          "signed_gap": -0.14878248035607888
        },
        {
          "bucket": "0.9-1.0",
          "sample_count": 4802,
          "average_confidence": 0.9666469068837037,
          "empirical_accuracy": 0.9989587671803415,
          "signed_gap": -0.032311860296637795
        }
      ],
      "selective_accuracy": [
        {
          "threshold": 0.5,
          "coverage": 0.8222667334334836,
          "accuracy": 0.911128253014494,
          "risk": 0.08887174698550604,
          "sample_count": 24631
        },
        {
          "threshold": 0.6,
          "coverage": 0.5693540310465699,
          "accuracy": 0.9902081501026092,
          "risk": 0.009791849897390792,
          "sample_count": 17055
        },
        {
          "threshold": 0.7,
          "coverage": 0.41919545985645135,
          "accuracy": 0.9968145257625229,
          "risk": 0.0031854742374770595,
          "sample_count": 12557
        },
        {
          "threshold": 0.8,
          "coverage": 0.27975296277749956,
          "accuracy": 0.9979713603818616,
          "risk": 0.0020286396181383726,
          "sample_count": 8380
        },
        {
          "threshold": 0.9,
          "coverage": 0.16030712735770322,
          "accuracy": 0.9989587671803415,
          "risk": 0.0010412328196585108,
          "sample_count": 4802
        },
        {
          "threshold": 0.95,
          "coverage": 0.10512435319646136,
          "accuracy": 0.9990473166084471,
          "risk": 0.0009526833915528554,
          "sample_count": 3149
        }
      ]
    },
    "by_kind": {
      "noul": {
        "count": 12229,
        "top1_accuracy": 0.8735791969907597,
        "nll": 0.4913208432770474,
        "brier_score": 0.003971773635485195,
        "ece": 0.14664666192959913,
        "signed_calibration_error": -0.14664666192959916,
        "kl_divergence": 0.0052616524114012615
      },
      "choice": {
        "count": 9199,
        "top1_accuracy": 0.8891183824328731,
        "nll": 0.7238766215674687,
        "brier_score": 0.024050878430150635,
        "ece": 0.19473382800328293,
        "signed_calibration_error": -0.1947338280032829,
        "kl_divergence": 0.048177643130987116
      },
      "score": {
        "count": 8527,
        "top1_accuracy": 0.8734607716664712,
        "nll": 1.0703111638621037,
        "brier_score": 0.010424387722836225,
        "ece": 0.32649771074931644,
        "signed_calibration_error": -0.3264977107493165,
        "kl_divergence": 0.026309488887190972
      }
    }
  },
  "ood": {
    "split": "ood",
    "temperature": 1.0087398771573446,
    "default_prediction": "raw",
    "calibration_comparison": {
      "raw": {
        "count": 13058,
        "top1_accuracy": 0.8785418900291009,
        "nll": 0.8813702719923984,
        "brier_score": 0.19665904522323993,
        "ece": 0.0934735002333655,
        "signed_calibration_error": 0.09338698955074462,
        "kl_divergence": 0.8662859112694271,
        "confidence_analysis": {
          "correct": {
            "count": 11472,
            "mean": 0.9861596955215062,
            "median": 0.9999983310699463,
            "std": 0.07204877546218105,
            "p25": 0.9999236762523651,
            "p75": 0.9999998807907104
          },
          "incorrect": {
            "count": 1586,
            "mean": 0.8689932424532811,
            "median": 0.9988584518432617,
            "std": 0.21735096923047184,
            "p25": 0.8063253164291382,
            "p75": 0.9999670684337616
          }
        },
        "confidence_histogram": [
          {
            "bucket": "0.0-0.1",
            "sample_count": 0,
            "average_confidence": null,
            "empirical_accuracy": null,
            "signed_gap": null
          },
          {
            "bucket": "0.1-0.2",
            "sample_count": 1,
            "average_confidence": 0.19825825095176697,
            "empirical_accuracy": 0.0,
            "signed_gap": 0.19825825095176697
          },
          {
            "bucket": "0.2-0.3",
            "sample_count": 65,
            "average_confidence": 0.256245233462407,
            "empirical_accuracy": 0.23076923076923078,
            "signed_gap": 0.025476002693176236
          },
          {
            "bucket": "0.3-0.4",
            "sample_count": 44,
            "average_confidence": 0.34126521985639224,
            "empirical_accuracy": 0.25,
            "signed_gap": 0.09126521985639224
          },
          {
            "bucket": "0.4-0.5",
            "sample_count": 25,
            "average_confidence": 0.4627038073539734,
            "empirical_accuracy": 0.52,
            "signed_gap": -0.057296192646026645
          },
          {
            "bucket": "0.5-0.6",
            "sample_count": 357,
            "average_confidence": 0.5382261586790326,
            "empirical_accuracy": 0.39775910364145656,
            "signed_gap": 0.140467055037576
          },
          {
            "bucket": "0.6-0.7",
            "sample_count": 103,
            "average_confidence": 0.6438586167918826,
            "empirical_accuracy": 0.5825242718446602,
            "signed_gap": 0.06133434494722245
          },
          {
            "bucket": "0.7-0.8",
            "sample_count": 100,
            "average_confidence": 0.7468274587392807,
            "empirical_accuracy": 0.61,
            "signed_gap": 0.1368274587392807
          },
          {
            "bucket": "0.8-0.9",
            "sample_count": 172,
            "average_confidence": 0.8525181381508361,
            "empirical_accuracy": 0.6046511627906976,
            "signed_gap": 0.24786697536013846
          },
          {
            "bucket": "0.9-1.0",
            "sample_count": 12191,
            "average_confidence": 0.998132212653288,
            "empirical_accuracy": 0.9077188089574276,
            "signed_gap": 0.09041340369586037
          }
        ],
        "selective_accuracy": [
          {
            "threshold": 0.5,
            "coverage": 0.989661510185327,
            "accuracy": 0.8847016946529443,
            "risk": 0.11529830534705565,
            "sample_count": 12923
          },
          {
            "threshold": 0.6,
            "coverage": 0.9623219482309695,
            "accuracy": 0.8985357313385326,
            "risk": 0.1014642686614674,
            "sample_count": 12566
          },
          {
            "threshold": 0.7,
            "coverage": 0.9544340634094042,
            "accuracy": 0.9011473962930273,
            "risk": 0.09885260370697269,
            "sample_count": 12463
          },
          {
            "threshold": 0.8,
            "coverage": 0.9467759228059427,
            "accuracy": 0.9035023861522284,
            "risk": 0.09649761384777156,
            "sample_count": 12363
          },
          {
            "threshold": 0.9,
            "coverage": 0.9336039209679889,
            "accuracy": 0.9077188089574276,
            "risk": 0.09228119104257237,
            "sample_count": 12191
          },
          {
            "threshold": 0.95,
            "coverage": 0.9241078266196967,
            "accuracy": 0.9104168393138311,
            "risk": 0.0895831606861689,
            "sample_count": 12067
          }
        ]
      },
      "temperature_scaled": {
        "count": 13058,
        "top1_accuracy": 0.8785418900291009,
        "nll": 0.8743518421149571,
        "brier_score": 0.19656952511113734,
        "ece": 0.09334952718772531,
        "signed_calibration_error": 0.09325994317919446,
        "kl_divergence": 0.8592674813919858,
        "confidence_analysis": {
          "correct": {
            "count": 11472,
            "mean": 0.9860722430341967,
            "median": 0.9999981217543632,
            "std": 0.07221466062590726,
            "p25": 0.999917101407375,
            "p75": 0.9999998629109141
          },
          "incorrect": {
            "count": 1586,
            "mean": 0.8685798019833653,
            "median": 0.9987841752508422,
            "std": 0.2175917495077864,
            "p25": 0.8042375494799926,
            "p75": 0.9999636629198553
          }
        },
        "confidence_histogram": [
          {
            "bucket": "0.0-0.1",
            "sample_count": 0,
            "average_confidence": null,
            "empirical_accuracy": null,
            "signed_gap": null
          },
          {
            "bucket": "0.1-0.2",
            "sample_count": 1,
            "average_confidence": 0.19798991716161005,
            "empirical_accuracy": 0.0,
            "signed_gap": 0.19798991716161005
          },
          {
            "bucket": "0.2-0.3",
            "sample_count": 67,
            "average_confidence": 0.25693683657772004,
            "empirical_accuracy": 0.22388059701492538,
            "signed_gap": 0.03305623956279466
          },
          {
            "bucket": "0.3-0.4",
            "sample_count": 42,
            "average_confidence": 0.3418979937441187,
            "empirical_accuracy": 0.2619047619047619,
            "signed_gap": 0.0799932318393568
          },
          {
            "bucket": "0.4-0.5",
            "sample_count": 25,
            "average_confidence": 0.4615572793674953,
            "empirical_accuracy": 0.52,
            "signed_gap": -0.05844272063250472
          },
          {
            "bucket": "0.5-0.6",
            "sample_count": 358,
            "average_confidence": 0.5379884025898151,
            "empirical_accuracy": 0.3994413407821229,
            "signed_gap": 0.13854706180769216
          },
          {
            "bucket": "0.6-0.7",
            "sample_count": 103,
            "average_confidence": 0.6434854397989195,
            "empirical_accuracy": 0.5825242718446602,
            "signed_gap": 0.06096116795425932
          },
          {
            "bucket": "0.7-0.8",
            "sample_count": 103,
            "average_confidence": 0.7474879529885827,
            "empirical_accuracy": 0.6019417475728155,
            "signed_gap": 0.14554620541576724
          },
          {
            "bucket": "0.8-0.9",
            "sample_count": 177,
            "average_confidence": 0.8541454633210329,
            "empirical_accuracy": 0.6045197740112994,
            "signed_gap": 0.2496256893097335
          },
          {
            "bucket": "0.9-1.0",
            "sample_count": 12182,
            "average_confidence": 0.9981466982229948,
            "empirical_accuracy": 0.9079789853882778,
            "signed_gap": 0.09016771283471703
          }
        ],
        "selective_accuracy": [
          {
            "threshold": 0.5,
            "coverage": 0.989661510185327,
            "accuracy": 0.8847016946529443,
            "risk": 0.11529830534705565,
            "sample_count": 12923
          },
          {
            "threshold": 0.6,
            "coverage": 0.9622453668249349,
            "accuracy": 0.8985276561878233,
            "risk": 0.10147234381217674,
            "sample_count": 12565
          },
          {
            "threshold": 0.7,
            "coverage": 0.9543574820033696,
            "accuracy": 0.9011394639704702,
            "risk": 0.09886053602952982,
            "sample_count": 12462
          },
          {
            "threshold": 0.8,
            "coverage": 0.9464695971818042,
            "accuracy": 0.9036329800145643,
            "risk": 0.09636701998543573,
            "sample_count": 12359
          },
          {
            "threshold": 0.9,
            "coverage": 0.9329146883136774,
            "accuracy": 0.9079789853882778,
            "risk": 0.09202101461172218,
            "sample_count": 12182
          },
          {
            "threshold": 0.95,
            "coverage": 0.9235717567774544,
            "accuracy": 0.9104477611940298,
            "risk": 0.08955223880597019,
            "sample_count": 12060
          }
        ]
      }
    },
    "overall": {
      "count": 13058,
      "top1_accuracy": 0.8785418900291009,
      "nll": 0.8813702719923984,
      "brier_score": 0.19665904522323993,
      "ece": 0.0934735002333655,
      "signed_calibration_error": 0.09338698955074462,
      "kl_divergence": 0.8662859112694271,
      "confidence_analysis": {
        "correct": {
          "count": 11472,
          "mean": 0.9861596955215062,
          "median": 0.9999983310699463,
          "std": 0.07204877546218105,
          "p25": 0.9999236762523651,
          "p75": 0.9999998807907104
        },
        "incorrect": {
          "count": 1586,
          "mean": 0.8689932424532811,
          "median": 0.9988584518432617,
          "std": 0.21735096923047184,
          "p25": 0.8063253164291382,
          "p75": 0.9999670684337616
        }
      },
      "confidence_histogram": [
        {
          "bucket": "0.0-0.1",
          "sample_count": 0,
          "average_confidence": null,
          "empirical_accuracy": null,
          "signed_gap": null
        },
        {
          "bucket": "0.1-0.2",
          "sample_count": 1,
          "average_confidence": 0.19825825095176697,
          "empirical_accuracy": 0.0,
          "signed_gap": 0.19825825095176697
        },
        {
          "bucket": "0.2-0.3",
          "sample_count": 65,
          "average_confidence": 0.256245233462407,
          "empirical_accuracy": 0.23076923076923078,
          "signed_gap": 0.025476002693176236
        },
        {
          "bucket": "0.3-0.4",
          "sample_count": 44,
          "average_confidence": 0.34126521985639224,
          "empirical_accuracy": 0.25,
          "signed_gap": 0.09126521985639224
        },
        {
          "bucket": "0.4-0.5",
          "sample_count": 25,
          "average_confidence": 0.4627038073539734,
          "empirical_accuracy": 0.52,
          "signed_gap": -0.057296192646026645
        },
        {
          "bucket": "0.5-0.6",
          "sample_count": 357,
          "average_confidence": 0.5382261586790326,
          "empirical_accuracy": 0.39775910364145656,
          "signed_gap": 0.140467055037576
        },
        {
          "bucket": "0.6-0.7",
          "sample_count": 103,
          "average_confidence": 0.6438586167918826,
          "empirical_accuracy": 0.5825242718446602,
          "signed_gap": 0.06133434494722245
        },
        {
          "bucket": "0.7-0.8",
          "sample_count": 100,
          "average_confidence": 0.7468274587392807,
          "empirical_accuracy": 0.61,
          "signed_gap": 0.1368274587392807
        },
        {
          "bucket": "0.8-0.9",
          "sample_count": 172,
          "average_confidence": 0.8525181381508361,
          "empirical_accuracy": 0.6046511627906976,
          "signed_gap": 0.24786697536013846
        },
        {
          "bucket": "0.9-1.0",
          "sample_count": 12191,
          "average_confidence": 0.998132212653288,
          "empirical_accuracy": 0.9077188089574276,
          "signed_gap": 0.09041340369586037
        }
      ],
      "selective_accuracy": [
        {
          "threshold": 0.5,
          "coverage": 0.989661510185327,
          "accuracy": 0.8847016946529443,
          "risk": 0.11529830534705565,
          "sample_count": 12923
        },
        {
          "threshold": 0.6,
          "coverage": 0.9623219482309695,
          "accuracy": 0.8985357313385326,
          "risk": 0.1014642686614674,
          "sample_count": 12566
        },
        {
          "threshold": 0.7,
          "coverage": 0.9544340634094042,
          "accuracy": 0.9011473962930273,
          "risk": 0.09885260370697269,
          "sample_count": 12463
        },
        {
          "threshold": 0.8,
          "coverage": 0.9467759228059427,
          "accuracy": 0.9035023861522284,
          "risk": 0.09649761384777156,
          "sample_count": 12363
        },
        {
          "threshold": 0.9,
          "coverage": 0.9336039209679889,
          "accuracy": 0.9077188089574276,
          "risk": 0.09228119104257237,
          "sample_count": 12191
        },
        {
          "threshold": 0.95,
          "coverage": 0.9241078266196967,
          "accuracy": 0.9104168393138311,
          "risk": 0.0895831606861689,
          "sample_count": 12067
        }
      ]
    },
    "by_kind": {
      "noul": {
        "count": 9767,
        "top1_accuracy": 0.9117436264973892,
        "nll": 0.6845514018050354,
        "brier_score": 0.1665514502307479,
        "ece": 0.07861264344307467,
        "signed_calibration_error": 0.07850355333421202,
        "kl_divergence": 0.6845514018050354
      },
      "choice": {
        "count": 3219,
        "top1_accuracy": 0.791860826343585,
        "nll": 1.4607612426226197,
        "brier_score": 0.2742619765350296,
        "ece": 0.13922969467819907,
        "signed_calibration_error": 0.13852634314944115,
        "kl_divergence": 1.399570940565907
      },
      "score": {
        "count": 72,
        "top1_accuracy": 0.25,
        "nll": 1.6767920867325716,
        "brier_score": 0.8113402174318239,
        "ece": 0.1267397738993168,
        "signed_calibration_error": 0.09426118806004524,
        "kl_divergence": 1.6767920867325716
      }
    }
  }
}