metrics.json

{
 "val": {
  "n": 11598,
  "top1": 0.4661148473874806,
  "top3": 0.6408001379548198,
  "division": 0.5929470598379031
 },
 "line_items": {
  "n": 194,
  "top1": 0.5968586387434555,
  "top3": 0.7591623036649214,
  "division": 0.7371134020618557
 },
 "manuals": {
  "n": 6890,
  "top1": 0.2797942689199118,
  "top3": 0.44584864070536373,
  "division": 0.46357039187227866
 },
 "manual_sections": {
  "sections": 153,
  "top1": 0.6066666666666667,
  "division": 0.8104575163398693
 },
 "txdot_bid_items": {
  "_doc": "790 TxDOT bid items labelled via txdot_crosswalk.json; capped = <=20 rows per TxDOT item (n=529); 95% Wilson CI",
  "transformer": {
   "raw": {
    "top1": 0.125,
    "top1_ci": [
     0.104,
     0.15
    ],
    "capped_top1": 0.161
   },
   "expanded": {
    "top1": 0.27,
    "top1_ci": [
     0.24,
     0.302
    ],
    "capped_top1": 0.272,
    "top3": 0.546,
    "division": 0.352
   }
  },
  "ensemble": {
   "raw": {
    "top1": 0.199,
    "top1_ci": [
     0.172,
     0.228
    ],
    "capped_top1": 0.223,
    "top3": 0.416,
    "division": 0.285
   },
   "expanded": {
    "top1": 0.489,
    "top1_ci": [
     0.454,
     0.523
    ],
    "capped_top1": 0.369,
    "capped_top1_ci": [
     0.329,
     0.411
    ],
    "top3": 0.671,
    "division": 0.57
   }
  }
 },
 "txdot_bid_items_bidmode": {
  "_doc": "ensemble/tfidf_bid.joblib (opt-in --bid-items); item-held-out = 5-fold CV grouped by TxDOT item number; paired vs default ensemble; building-set regression with the shipped model",
  "item_held_out": {
   "n": 790,
   "top1": 0.6076,
   "ci": [
    0.573,
    0.641
   ],
   "vs_published": {
    "new_right_old_wrong": 104,
    "old_right_new_wrong": 10,
    "p": 0.0
   }
  },
  "item_held_out_capped": {
   "n": 529,
   "top1": 0.4915,
   "ci": [
    0.449,
    0.534
   ],
   "vs_published": {
    "new_right_old_wrong": 73,
    "old_right_new_wrong": 8,
    "p": 0.0
   }
  },
  "default_same_rows": {
   "n": 790,
   "top1": 0.4886,
   "ci": [
    0.454,
    0.523
   ]
  },
  "items_seen_optimistic": 0.9532,
  "building_regression": {
   "val": {
    "default": 0.5875,
    "bid_mode": 0.5836
   },
   "line_items": {
    "default": 0.6907,
    "bid_mode": 0.701
   },
   "manuals": {
    "default": 0.3261,
    "bid_mode": 0.3287
   },
   "manual_sections": {
    "default": {
     "sections": 153,
     "top1": 0.62,
     "division": 0.803921568627451
    },
    "bid_mode": {
     "sections": 153,
     "top1": 0.6133333333333333,
     "division": 0.7908496732026143
    }
   }
  },
  "train_items": 2842,
  "bid_weight": 3.0
 }
}