{
  "n": 200,
  "images": 199,
  "accuracy_all": 0.62,
  "accuracy": {
    "noul": null,
    "choice": 0.62
  },
  "nll": 0.9363848029670663,
  "brier": 0.49875960038674777,
  "ece": 0.08038865998387337,
  "ece_bins": 15,
  "reliability": [
    {
      "bin": [
        0.2,
        0.3
      ],
      "n": 4,
      "confidence": 0.28683528304100037,
      "accuracy": 0.25
    },
    {
      "bin": [
        0.3,
        0.4
      ],
      "n": 21,
      "confidence": 0.3550230505920592,
      "accuracy": 0.23809523809523808
    },
    {
      "bin": [
        0.4,
        0.5
      ],
      "n": 27,
      "confidence": 0.45185136574286,
      "accuracy": 0.4074074074074074
    },
    {
      "bin": [
        0.5,
        0.6
      ],
      "n": 23,
      "confidence": 0.5413056534269581,
      "accuracy": 0.5217391304347826
    },
    {
      "bin": [
        0.6,
        0.7
      ],
      "n": 23,
      "confidence": 0.6432638349740402,
      "accuracy": 0.6086956521739131
    },
    {
      "bin": [
        0.7,
        0.8
      ],
      "n": 24,
      "confidence": 0.750172791381677,
      "accuracy": 0.5416666666666666
    },
    {
      "bin": [
        0.8,
        0.9
      ],
      "n": 26,
      "confidence": 0.8457011099045093,
      "accuracy": 0.7307692307692307
    },
    {
      "bin": [
        0.9,
        1.0
      ],
      "n": 52,
      "confidence": 0.9622523956573926,
      "accuracy": 0.9423076923076923
    }
  ],
  "thresholds": {
    "0.6": {
      "coverage": 0.625,
      "accuracy": 0.76
    },
    "0.7": {
      "coverage": 0.51,
      "accuracy": 0.7941176470588235
    },
    "0.8": {
      "coverage": 0.39,
      "accuracy": 0.8717948717948718
    },
    "0.9": {
      "coverage": 0.26,
      "accuracy": 0.9423076923076923
    }
  },
  "by_source": {
    "aokvqa": {
      "n": 200,
      "images": 199,
      "accuracy": 0.62,
      "nll": 0.9363848029670663,
      "ece": 0.08038865998387337
    }
  },
  "by_kind": {
    "aokvqa": {
      "n": 200,
      "images": 199,
      "accuracy": 0.62,
      "nll": 0.9363848029670663,
      "ece": 0.08038865998387337
    }
  },
  "by_axis": {
    "-": {
      "n": 200,
      "images": 199,
      "accuracy": 0.62,
      "nll": 0.9363848029670663,
      "ece": 0.08038865998387337
    }
  },
  "by_options": {
    "4": {
      "n": 200,
      "images": 199,
      "accuracy": 0.62,
      "nll": 0.9363848029670663,
      "ece": 0.08038865998387337
    }
  },
  "ms_per_question": {
    "p50": 300.66265849995943,
    "p95": 321.46255515008306,
    "overall": 302.3525179750118
  },
  "request_latency_ms": {
    "p50": 300.66265849995943,
    "p95": 321.46255515008306,
    "requests": 200,
    "warmup_excluded": 3
  },
  "questions_per_request": {
    "mean": 1.0,
    "p50": 1.0,
    "max": 1
  },
  "latency_ms": {
    "p50": 300.66265849995943,
    "p95": 321.46255515008306,
    "requests": 200,
    "warmup_excluded": 3
  },
  "control": "none",
  "model": "devision-v0.3",
  "reading": {
    "mode": "none",
    "routed_questions": 0,
    "read_questions": 0
  },
  "temperature": {
    "choice": 1.160062551498413,
    "noul": 0.9890053272247314,
    "by_options": {
      "choice:2": 1.1245635747909546,
      "choice:3-5": 1.1677029132843018,
      "noul:2": 0.9890053272247314
    }
  },
  "grouping": {
    "max_questions_per_request": 1,
    "by": "none (one question per request)",
    "max_questions_per_call": "one request per call"
  },
  "environment": {
    "threads": 4,
    "machine": "x86_64",
    "system": "Linux"
  },
  "run": {
    "checkpoint": "runs/r24-v05/train",
    "data": "runs/v04-cpu/latency200.jsonl",
    "data_sha256": "e0f4d5b0f82ff59ba91d141f49833ad9a097ac36842b98c32e2c4a834d0b9691",
    "role": "monitoring",
    "device": "cpu",
    "temperature": "fitted",
    "control": "none",
    "seed": 0,
    "code": "143d23b+dirty",
    "reading": {
      "mode": "none",
      "routed_questions": 0,
      "read_questions": 0
    },
    "code_identity": {
      "modules": {
        "devision": {
          "path": "/home/bruce/proj/devision/src/devision/__init__.py",
          "sha256": "aa00e0d329f33444a51bc4cab7691911b6b226664fdc2fbd438103de8d0c27a1"
        },
        "devision.image": {
          "path": "/home/bruce/proj/devision/src/devision/image.py",
          "sha256": "1ae756d80aa9ccaece93b82eb53dcd65559a2d11f951012ad3ad125f1de81c99"
        },
        "devision.infer": {
          "path": "/home/bruce/proj/devision/src/devision/infer.py",
          "sha256": "297ca0790b8d2d77711a13a3024a78a50b5722935bb38519fd5ae5d7672c115f"
        },
        "devision.model": {
          "path": "/home/bruce/proj/devision/src/devision/model.py",
          "sha256": "e4f53a944c5c0fc6f4bb30e518af2fc91eec2ece78eb88ff88651323fb9084e8"
        },
        "devision.reading": {
          "path": "/home/bruce/proj/devision/src/devision/reading.py",
          "sha256": "dc2f3cc2151ec4761dcfc404b28d609a932aac70ff0ff02b0dd6e170b6c1f8ba"
        },
        "devision.train": {
          "path": "/home/bruce/proj/devision/src/devision/train/__init__.py",
          "sha256": "16c32ba2bae66029fb0afd4946827c475729aa45c7dde335864d002c564df289"
        },
        "devision.train.cli": {
          "path": "/home/bruce/proj/devision/src/devision/train/cli.py",
          "sha256": "aab8811a63c89726fd78dd7ea88ab930c6d115b529a011173ccc8637f081d342"
        },
        "devision.train.evaluate": {
          "path": "/home/bruce/proj/devision/src/devision/train/evaluate.py",
          "sha256": "4d5df39b37b0854ead1b8c7d0a04933e24ab9b05e0e341f0bdbfbc6911af2d90"
        },
        "devision.train.samples": {
          "path": "/home/bruce/proj/devision/src/devision/train/samples.py",
          "sha256": "5a4d5a75ec31380f5470a22c0ba4e370e170ba0499e5d4a77ed573450f9c3e31"
        }
      },
      "version": "0.3.0.dev1",
      "python": "3.12.3"
    }
  }
}