| { |
| "scope": "Existing validation distribution; threshold selection, not fresh final acceptance.", |
| "comparison": "Two complete configurations on the same validation rows: B2 joint encoding with six-word tool options; B5 independent encoding with up to 60-word options. Corpus hashes intentionally differ.", |
| "transformation": "Per-tool breakdowns omitted; all aggregate threshold rows retained without recalculation.", |
| "prediction_metadata_sha256": "de8a6974d1ca4a615f319089b2a6b023e5e57ede9bc03e52de5aa9524bde11c9", |
| "reports": { |
| "B2": { |
| "original_report_sha256": "4b705b7e02a4307e3d23898691c6e82fc72fcfbe9d4e94c0c0341838af21d206", |
| "input_sha256": { |
| "predictions": "b20f3ddf176f4a5905ac1a4a3073e41a1c4833a930292c942abbba974c10c9b1", |
| "rows": "67c4c86e3385f0241048f2633d97a4aad78a53a3b7a540592937a20eba669fd3", |
| "corpus": "51e3a65258fc34581bc8cfaae0f7c97f444f0a50f3dda906c77b6554fd5c66d3" |
| }, |
| "prediction_metadata_sha256": "e8d91777999dded4eafe02cf46d7e99a3a89b0bd376c7f6922519e33f7e360dd", |
| "encoding": "joint", |
| "tool_option_words": 6, |
| "head_sha256": "bc432ea7497e9300e39230ee9850de8f42e3f9a3cbbf372259263c49c77011e4", |
| "truth": { |
| "consensus": [ |
| { |
| "threshold": 0.5, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 935, |
| "empty": 128, |
| "needed_turns": 216, |
| "any_needed_loaded": 173, |
| "all_needed_loaded": 149, |
| "tn": 29289, |
| "fp": 509, |
| "tp": 203, |
| "unknown_loaded": 223, |
| "fn": 82 |
| }, |
| "precision": 0.2851123595505618, |
| "recall": 0.712280701754386, |
| "f0.5": 0.3239706351739547, |
| "loads_per_turn": 2.101123595505618, |
| "empty_turn_rate": 0.2876404494382023 |
| }, |
| { |
| "threshold": 0.6, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 734, |
| "empty": 158, |
| "needed_turns": 216, |
| "any_needed_loaded": 159, |
| "all_needed_loaded": 137, |
| "tn": 29438, |
| "fp": 360, |
| "tp": 186, |
| "unknown_loaded": 188, |
| "fn": 99 |
| }, |
| "precision": 0.34065934065934067, |
| "recall": 0.6526315789473685, |
| "f0.5": 0.37667071688942894, |
| "loads_per_turn": 1.649438202247191, |
| "empty_turn_rate": 0.3550561797752809 |
| }, |
| { |
| "threshold": 0.7, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 533, |
| "empty": 189, |
| "needed_turns": 216, |
| "any_needed_loaded": 147, |
| "all_needed_loaded": 122, |
| "tn": 29576, |
| "fp": 222, |
| "tp": 166, |
| "unknown_loaded": 145, |
| "fn": 119 |
| }, |
| "precision": 0.42783505154639173, |
| "recall": 0.5824561403508772, |
| "f0.5": 0.4518236254763201, |
| "loads_per_turn": 1.1977528089887641, |
| "empty_turn_rate": 0.4247191011235955 |
| }, |
| { |
| "threshold": 0.75, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 419, |
| "empty": 220, |
| "needed_turns": 216, |
| "any_needed_loaded": 131, |
| "all_needed_loaded": 105, |
| "tn": 29640, |
| "fp": 158, |
| "tp": 147, |
| "unknown_loaded": 114, |
| "fn": 138 |
| }, |
| "precision": 0.4819672131147541, |
| "recall": 0.5157894736842106, |
| "f0.5": 0.4883720930232558, |
| "loads_per_turn": 0.9415730337078652, |
| "empty_turn_rate": 0.4943820224719101 |
| }, |
| { |
| "threshold": 0.8, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 301, |
| "empty": 262, |
| "needed_turns": 216, |
| "any_needed_loaded": 108, |
| "all_needed_loaded": 83, |
| "tn": 29703, |
| "fp": 95, |
| "tp": 117, |
| "unknown_loaded": 89, |
| "fn": 168 |
| }, |
| "precision": 0.5518867924528302, |
| "recall": 0.4105263157894737, |
| "f0.5": 0.5163283318623124, |
| "loads_per_turn": 0.6764044943820224, |
| "empty_turn_rate": 0.5887640449438202 |
| }, |
| { |
| "threshold": 0.85, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 211, |
| "empty": 299, |
| "needed_turns": 216, |
| "any_needed_loaded": 93, |
| "all_needed_loaded": 72, |
| "tn": 29745, |
| "fp": 53, |
| "tp": 100, |
| "unknown_loaded": 58, |
| "fn": 185 |
| }, |
| "precision": 0.6535947712418301, |
| "recall": 0.3508771929824561, |
| "f0.5": 0.5574136008918618, |
| "loads_per_turn": 0.47415730337078654, |
| "empty_turn_rate": 0.6719101123595506 |
| }, |
| { |
| "threshold": 0.89, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 151, |
| "empty": 326, |
| "needed_turns": 216, |
| "any_needed_loaded": 80, |
| "all_needed_loaded": 60, |
| "tn": 29771, |
| "fp": 27, |
| "fn": 201, |
| "unknown_loaded": 40, |
| "tp": 84 |
| }, |
| "precision": 0.7567567567567568, |
| "recall": 0.29473684210526313, |
| "f0.5": 0.5761316872427984, |
| "loads_per_turn": 0.3393258426966292, |
| "empty_turn_rate": 0.7325842696629213 |
| }, |
| { |
| "threshold": 0.9, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 138, |
| "empty": 336, |
| "needed_turns": 216, |
| "any_needed_loaded": 74, |
| "all_needed_loaded": 56, |
| "tn": 29773, |
| "fp": 25, |
| "fn": 207, |
| "unknown_loaded": 35, |
| "tp": 78 |
| }, |
| "precision": 0.7572815533980582, |
| "recall": 0.2736842105263158, |
| "f0.5": 0.5595408895265424, |
| "loads_per_turn": 0.3101123595505618, |
| "empty_turn_rate": 0.755056179775281 |
| }, |
| { |
| "threshold": 0.95, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 84, |
| "empty": 373, |
| "needed_turns": 216, |
| "any_needed_loaded": 51, |
| "all_needed_loaded": 43, |
| "tn": 29789, |
| "fn": 230, |
| "unknown_loaded": 20, |
| "tp": 55, |
| "fp": 9 |
| }, |
| "precision": 0.859375, |
| "recall": 0.19298245614035087, |
| "f0.5": 0.5083179297597042, |
| "loads_per_turn": 0.18876404494382024, |
| "empty_turn_rate": 0.8382022471910112 |
| }, |
| { |
| "threshold": 0.97, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 62, |
| "empty": 391, |
| "needed_turns": 216, |
| "any_needed_loaded": 41, |
| "all_needed_loaded": 34, |
| "tn": 29793, |
| "fn": 241, |
| "unknown_loaded": 13, |
| "tp": 44, |
| "fp": 5 |
| }, |
| "precision": 0.8979591836734694, |
| "recall": 0.1543859649122807, |
| "f0.5": 0.4573804573804574, |
| "loads_per_turn": 0.1393258426966292, |
| "empty_turn_rate": 0.8786516853932584 |
| }, |
| { |
| "threshold": 0.99, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 29, |
| "empty": 417, |
| "needed_turns": 216, |
| "any_needed_loaded": 21, |
| "all_needed_loaded": 15, |
| "tn": 29795, |
| "fn": 264, |
| "unknown_loaded": 5, |
| "tp": 21, |
| "fp": 3 |
| }, |
| "precision": 0.875, |
| "recall": 0.07368421052631578, |
| "f0.5": 0.2755905511811024, |
| "loads_per_turn": 0.0651685393258427, |
| "empty_turn_rate": 0.9370786516853933 |
| } |
| ], |
| "called": [ |
| { |
| "threshold": 0.5, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 935, |
| "empty": 128, |
| "needed_turns": 228, |
| "any_needed_loaded": 182, |
| "all_needed_loaded": 91, |
| "tn": 29888, |
| "fp": 659, |
| "tp": 276, |
| "fn": 248 |
| }, |
| "precision": 0.29518716577540105, |
| "recall": 0.5267175572519084, |
| "f0.5": 0.3236397748592871, |
| "loads_per_turn": 2.101123595505618, |
| "empty_turn_rate": 0.2876404494382023 |
| }, |
| { |
| "threshold": 0.6, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 734, |
| "empty": 158, |
| "needed_turns": 228, |
| "any_needed_loaded": 166, |
| "all_needed_loaded": 76, |
| "tn": 30053, |
| "fp": 494, |
| "tp": 240, |
| "fn": 284 |
| }, |
| "precision": 0.32697547683923706, |
| "recall": 0.4580152671755725, |
| "f0.5": 0.3468208092485549, |
| "loads_per_turn": 1.649438202247191, |
| "empty_turn_rate": 0.3550561797752809 |
| }, |
| { |
| "threshold": 0.7, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 533, |
| "empty": 189, |
| "needed_turns": 228, |
| "any_needed_loaded": 149, |
| "all_needed_loaded": 59, |
| "tn": 30208, |
| "tp": 194, |
| "fp": 339, |
| "fn": 330 |
| }, |
| "precision": 0.36397748592870544, |
| "recall": 0.3702290076335878, |
| "f0.5": 0.36521084337349397, |
| "loads_per_turn": 1.1977528089887641, |
| "empty_turn_rate": 0.4247191011235955 |
| }, |
| { |
| "threshold": 0.75, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 419, |
| "empty": 220, |
| "needed_turns": 228, |
| "any_needed_loaded": 133, |
| "all_needed_loaded": 46, |
| "tn": 30286, |
| "fn": 366, |
| "fp": 261, |
| "tp": 158 |
| }, |
| "precision": 0.37708830548926014, |
| "recall": 0.3015267175572519, |
| "f0.5": 0.35909090909090907, |
| "loads_per_turn": 0.9415730337078652, |
| "empty_turn_rate": 0.4943820224719101 |
| }, |
| { |
| "threshold": 0.8, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 301, |
| "empty": 262, |
| "needed_turns": 228, |
| "any_needed_loaded": 106, |
| "all_needed_loaded": 38, |
| "tn": 30368, |
| "fn": 402, |
| "fp": 179, |
| "tp": 122 |
| }, |
| "precision": 0.4053156146179402, |
| "recall": 0.23282442748091603, |
| "f0.5": 0.35300925925925924, |
| "loads_per_turn": 0.6764044943820224, |
| "empty_turn_rate": 0.5887640449438202 |
| }, |
| { |
| "threshold": 0.85, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 211, |
| "empty": 299, |
| "needed_turns": 228, |
| "any_needed_loaded": 83, |
| "all_needed_loaded": 32, |
| "tn": 30430, |
| "fn": 430, |
| "fp": 117, |
| "tp": 94 |
| }, |
| "precision": 0.44549763033175355, |
| "recall": 0.17938931297709923, |
| "f0.5": 0.3435672514619883, |
| "loads_per_turn": 0.47415730337078654, |
| "empty_turn_rate": 0.6719101123595506 |
| }, |
| { |
| "threshold": 0.89, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 151, |
| "empty": 326, |
| "needed_turns": 228, |
| "any_needed_loaded": 72, |
| "all_needed_loaded": 24, |
| "tn": 30474, |
| "fn": 446, |
| "fp": 73, |
| "tp": 78 |
| }, |
| "precision": 0.5165562913907285, |
| "recall": 0.14885496183206107, |
| "f0.5": 0.34574468085106386, |
| "loads_per_turn": 0.3393258426966292, |
| "empty_turn_rate": 0.7325842696629213 |
| }, |
| { |
| "threshold": 0.9, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 138, |
| "empty": 336, |
| "needed_turns": 228, |
| "any_needed_loaded": 65, |
| "all_needed_loaded": 22, |
| "tn": 30479, |
| "fn": 454, |
| "fp": 68, |
| "tp": 70 |
| }, |
| "precision": 0.5072463768115942, |
| "recall": 0.13358778625954199, |
| "f0.5": 0.3252788104089219, |
| "loads_per_turn": 0.3101123595505618, |
| "empty_turn_rate": 0.755056179775281 |
| }, |
| { |
| "threshold": 0.95, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 84, |
| "empty": 373, |
| "needed_turns": 228, |
| "any_needed_loaded": 42, |
| "all_needed_loaded": 15, |
| "tn": 30508, |
| "fn": 479, |
| "tp": 45, |
| "fp": 39 |
| }, |
| "precision": 0.5357142857142857, |
| "recall": 0.08587786259541985, |
| "f0.5": 0.2616279069767442, |
| "loads_per_turn": 0.18876404494382024, |
| "empty_turn_rate": 0.8382022471910112 |
| }, |
| { |
| "threshold": 0.97, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 62, |
| "empty": 391, |
| "needed_turns": 228, |
| "any_needed_loaded": 32, |
| "all_needed_loaded": 9, |
| "tn": 30520, |
| "fn": 489, |
| "tp": 35, |
| "fp": 27 |
| }, |
| "precision": 0.5645161290322581, |
| "recall": 0.06679389312977099, |
| "f0.5": 0.2266839378238342, |
| "loads_per_turn": 0.1393258426966292, |
| "empty_turn_rate": 0.8786516853932584 |
| }, |
| { |
| "threshold": 0.99, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 29, |
| "empty": 417, |
| "needed_turns": 228, |
| "any_needed_loaded": 15, |
| "all_needed_loaded": 5, |
| "tn": 30533, |
| "fn": 509, |
| "tp": 15, |
| "fp": 14 |
| }, |
| "precision": 0.5172413793103449, |
| "recall": 0.02862595419847328, |
| "f0.5": 0.1171875, |
| "loads_per_turn": 0.0651685393258427, |
| "empty_turn_rate": 0.9370786516853933 |
| } |
| ] |
| } |
| }, |
| "B5": { |
| "original_report_sha256": "18a25333e07dfb5b79c6709cf7c99855517ce07e51e3cd80ccc8e15e8a656a51", |
| "input_sha256": { |
| "predictions": "e70357d58524a0714882aef5c5de98a6c4cf74a3ad82ae23ee1ed9f97789d1c0", |
| "rows": "67c4c86e3385f0241048f2633d97a4aad78a53a3b7a540592937a20eba669fd3", |
| "corpus": "c77abd8b40965c05cc43a7545e8c2db2405bdcfc18a73bad254632d523b67bcd" |
| }, |
| "prediction_metadata_sha256": "de8a6974d1ca4a615f319089b2a6b023e5e57ede9bc03e52de5aa9524bde11c9", |
| "encoding": "independent", |
| "tool_option_words": 60, |
| "head_sha256": "550f30b94d5c6a8680d2cb2a607ebc9ee33e709c7679f72606d1e9fda17f2e90", |
| "truth": { |
| "consensus": [ |
| { |
| "threshold": 0.5, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 639, |
| "empty": 176, |
| "needed_turns": 216, |
| "any_needed_loaded": 166, |
| "all_needed_loaded": 141, |
| "tn": 29523, |
| "fp": 275, |
| "tp": 197, |
| "unknown_loaded": 167, |
| "fn": 88 |
| }, |
| "precision": 0.4173728813559322, |
| "recall": 0.6912280701754386, |
| "f0.5": 0.45329038196042337, |
| "loads_per_turn": 1.4359550561797754, |
| "empty_turn_rate": 0.3955056179775281 |
| }, |
| { |
| "threshold": 0.6, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 539, |
| "empty": 191, |
| "needed_turns": 216, |
| "any_needed_loaded": 164, |
| "all_needed_loaded": 138, |
| "tn": 29594, |
| "fp": 204, |
| "tp": 192, |
| "unknown_loaded": 143, |
| "fn": 93 |
| }, |
| "precision": 0.48484848484848486, |
| "recall": 0.6736842105263158, |
| "f0.5": 0.5136436597110754, |
| "loads_per_turn": 1.2112359550561798, |
| "empty_turn_rate": 0.42921348314606744 |
| }, |
| { |
| "threshold": 0.7, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 429, |
| "empty": 222, |
| "needed_turns": 216, |
| "any_needed_loaded": 145, |
| "all_needed_loaded": 118, |
| "tn": 29653, |
| "fp": 145, |
| "tp": 167, |
| "unknown_loaded": 117, |
| "fn": 118 |
| }, |
| "precision": 0.5352564102564102, |
| "recall": 0.5859649122807018, |
| "f0.5": 0.5446836268754077, |
| "loads_per_turn": 0.9640449438202248, |
| "empty_turn_rate": 0.49887640449438203 |
| }, |
| { |
| "threshold": 0.75, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 371, |
| "empty": 238, |
| "needed_turns": 216, |
| "any_needed_loaded": 136, |
| "all_needed_loaded": 108, |
| "tn": 29690, |
| "fp": 108, |
| "tp": 156, |
| "unknown_loaded": 107, |
| "fn": 129 |
| }, |
| "precision": 0.5909090909090909, |
| "recall": 0.5473684210526316, |
| "f0.5": 0.5816554809843401, |
| "loads_per_turn": 0.8337078651685393, |
| "empty_turn_rate": 0.5348314606741573 |
| }, |
| { |
| "threshold": 0.8, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 307, |
| "empty": 261, |
| "needed_turns": 216, |
| "any_needed_loaded": 126, |
| "all_needed_loaded": 100, |
| "tn": 29720, |
| "fp": 78, |
| "tp": 142, |
| "unknown_loaded": 87, |
| "fn": 143 |
| }, |
| "precision": 0.6454545454545455, |
| "recall": 0.4982456140350877, |
| "f0.5": 0.6094420600858369, |
| "loads_per_turn": 0.6898876404494382, |
| "empty_turn_rate": 0.5865168539325842 |
| }, |
| { |
| "threshold": 0.85, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 267, |
| "empty": 278, |
| "needed_turns": 216, |
| "any_needed_loaded": 118, |
| "all_needed_loaded": 91, |
| "tn": 29734, |
| "fp": 64, |
| "tp": 131, |
| "unknown_loaded": 72, |
| "fn": 154 |
| }, |
| "precision": 0.6717948717948717, |
| "recall": 0.45964912280701753, |
| "f0.5": 0.6150234741784038, |
| "loads_per_turn": 0.6, |
| "empty_turn_rate": 0.6247191011235955 |
| }, |
| { |
| "threshold": 0.89, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 222, |
| "empty": 295, |
| "needed_turns": 216, |
| "any_needed_loaded": 104, |
| "all_needed_loaded": 78, |
| "tn": 29754, |
| "fp": 44, |
| "tp": 115, |
| "unknown_loaded": 63, |
| "fn": 170 |
| }, |
| "precision": 0.7232704402515723, |
| "recall": 0.40350877192982454, |
| "f0.5": 0.6243213897937026, |
| "loads_per_turn": 0.49887640449438203, |
| "empty_turn_rate": 0.6629213483146067 |
| }, |
| { |
| "threshold": 0.9, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 216, |
| "empty": 297, |
| "needed_turns": 216, |
| "any_needed_loaded": 101, |
| "all_needed_loaded": 75, |
| "tn": 29757, |
| "fp": 41, |
| "tp": 112, |
| "unknown_loaded": 63, |
| "fn": 173 |
| }, |
| "precision": 0.7320261437908496, |
| "recall": 0.3929824561403509, |
| "f0.5": 0.6243032329988851, |
| "loads_per_turn": 0.4853932584269663, |
| "empty_turn_rate": 0.6674157303370787 |
| }, |
| { |
| "threshold": 0.95, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 137, |
| "empty": 338, |
| "needed_turns": 216, |
| "any_needed_loaded": 77, |
| "all_needed_loaded": 53, |
| "tn": 29780, |
| "fp": 18, |
| "fn": 202, |
| "unknown_loaded": 36, |
| "tp": 83 |
| }, |
| "precision": 0.8217821782178217, |
| "recall": 0.2912280701754386, |
| "f0.5": 0.602322206095791, |
| "loads_per_turn": 0.30786516853932583, |
| "empty_turn_rate": 0.7595505617977528 |
| }, |
| { |
| "threshold": 0.97, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 85, |
| "empty": 374, |
| "needed_turns": 216, |
| "any_needed_loaded": 51, |
| "all_needed_loaded": 37, |
| "tn": 29788, |
| "fp": 10, |
| "fn": 231, |
| "unknown_loaded": 21, |
| "tp": 54 |
| }, |
| "precision": 0.84375, |
| "recall": 0.18947368421052632, |
| "f0.5": 0.49907578558225507, |
| "loads_per_turn": 0.19101123595505617, |
| "empty_turn_rate": 0.8404494382022472 |
| }, |
| { |
| "threshold": 0.99, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 12, |
| "empty": 433, |
| "needed_turns": 216, |
| "any_needed_loaded": 9, |
| "all_needed_loaded": 5, |
| "tn": 29798, |
| "fn": 276, |
| "unknown_loaded": 3, |
| "tp": 9 |
| }, |
| "precision": 1.0, |
| "recall": 0.031578947368421054, |
| "f0.5": 0.14018691588785046, |
| "loads_per_turn": 0.02696629213483146, |
| "empty_turn_rate": 0.9730337078651685 |
| } |
| ], |
| "called": [ |
| { |
| "threshold": 0.5, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 639, |
| "empty": 176, |
| "needed_turns": 228, |
| "any_needed_loaded": 170, |
| "all_needed_loaded": 74, |
| "tn": 30152, |
| "fn": 280, |
| "fp": 395, |
| "tp": 244 |
| }, |
| "precision": 0.38184663536776214, |
| "recall": 0.46564885496183206, |
| "f0.5": 0.3961038961038961, |
| "loads_per_turn": 1.4359550561797754, |
| "empty_turn_rate": 0.3955056179775281 |
| }, |
| { |
| "threshold": 0.6, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 539, |
| "empty": 191, |
| "needed_turns": 228, |
| "any_needed_loaded": 165, |
| "all_needed_loaded": 68, |
| "tn": 30235, |
| "fn": 297, |
| "fp": 312, |
| "tp": 227 |
| }, |
| "precision": 0.42115027829313545, |
| "recall": 0.43320610687022904, |
| "f0.5": 0.42350746268656714, |
| "loads_per_turn": 1.2112359550561798, |
| "empty_turn_rate": 0.42921348314606744 |
| }, |
| { |
| "threshold": 0.7, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 429, |
| "empty": 222, |
| "needed_turns": 228, |
| "any_needed_loaded": 148, |
| "all_needed_loaded": 57, |
| "tn": 30312, |
| "fn": 330, |
| "fp": 235, |
| "tp": 194 |
| }, |
| "precision": 0.4522144522144522, |
| "recall": 0.3702290076335878, |
| "f0.5": 0.4330357142857143, |
| "loads_per_turn": 0.9640449438202248, |
| "empty_turn_rate": 0.49887640449438203 |
| }, |
| { |
| "threshold": 0.75, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 371, |
| "empty": 238, |
| "needed_turns": 228, |
| "any_needed_loaded": 141, |
| "all_needed_loaded": 51, |
| "tn": 30354, |
| "fn": 346, |
| "fp": 193, |
| "tp": 178 |
| }, |
| "precision": 0.4797843665768194, |
| "recall": 0.33969465648854963, |
| "f0.5": 0.44322709163346613, |
| "loads_per_turn": 0.8337078651685393, |
| "empty_turn_rate": 0.5348314606741573 |
| }, |
| { |
| "threshold": 0.8, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 307, |
| "empty": 261, |
| "needed_turns": 228, |
| "any_needed_loaded": 129, |
| "all_needed_loaded": 45, |
| "tn": 30395, |
| "fn": 369, |
| "fp": 152, |
| "tp": 155 |
| }, |
| "precision": 0.504885993485342, |
| "recall": 0.2958015267175573, |
| "f0.5": 0.442351598173516, |
| "loads_per_turn": 0.6898876404494382, |
| "empty_turn_rate": 0.5865168539325842 |
| }, |
| { |
| "threshold": 0.85, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 267, |
| "empty": 278, |
| "needed_turns": 228, |
| "any_needed_loaded": 116, |
| "all_needed_loaded": 36, |
| "tn": 30417, |
| "fn": 387, |
| "fp": 130, |
| "tp": 137 |
| }, |
| "precision": 0.5131086142322098, |
| "recall": 0.26145038167938933, |
| "f0.5": 0.43027638190954776, |
| "loads_per_turn": 0.6, |
| "empty_turn_rate": 0.6247191011235955 |
| }, |
| { |
| "threshold": 0.89, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 222, |
| "empty": 295, |
| "needed_turns": 228, |
| "any_needed_loaded": 105, |
| "all_needed_loaded": 31, |
| "tn": 30444, |
| "fn": 405, |
| "fp": 103, |
| "tp": 119 |
| }, |
| "precision": 0.536036036036036, |
| "recall": 0.22709923664122136, |
| "f0.5": 0.4213881019830028, |
| "loads_per_turn": 0.49887640449438203, |
| "empty_turn_rate": 0.6629213483146067 |
| }, |
| { |
| "threshold": 0.9, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 216, |
| "empty": 297, |
| "needed_turns": 228, |
| "any_needed_loaded": 103, |
| "all_needed_loaded": 31, |
| "tn": 30448, |
| "fn": 407, |
| "fp": 99, |
| "tp": 117 |
| }, |
| "precision": 0.5416666666666666, |
| "recall": 0.22328244274809161, |
| "f0.5": 0.4214697406340058, |
| "loads_per_turn": 0.4853932584269663, |
| "empty_turn_rate": 0.6674157303370787 |
| }, |
| { |
| "threshold": 0.95, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 137, |
| "empty": 338, |
| "needed_turns": 228, |
| "any_needed_loaded": 71, |
| "all_needed_loaded": 19, |
| "tn": 30485, |
| "fn": 449, |
| "fp": 62, |
| "tp": 75 |
| }, |
| "precision": 0.5474452554744526, |
| "recall": 0.1431297709923664, |
| "f0.5": 0.3498134328358209, |
| "loads_per_turn": 0.30786516853932583, |
| "empty_turn_rate": 0.7595505617977528 |
| }, |
| { |
| "threshold": 0.97, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 85, |
| "empty": 374, |
| "needed_turns": 228, |
| "any_needed_loaded": 45, |
| "all_needed_loaded": 11, |
| "tn": 30511, |
| "fn": 475, |
| "fp": 36, |
| "tp": 49 |
| }, |
| "precision": 0.5764705882352941, |
| "recall": 0.09351145038167939, |
| "f0.5": 0.2835648148148148, |
| "loads_per_turn": 0.19101123595505617, |
| "empty_turn_rate": 0.8404494382022472 |
| }, |
| { |
| "threshold": 0.99, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 12, |
| "empty": 433, |
| "needed_turns": 228, |
| "any_needed_loaded": 7, |
| "all_needed_loaded": 1, |
| "tn": 30542, |
| "fn": 517, |
| "fp": 5, |
| "tp": 7 |
| }, |
| "precision": 0.5833333333333334, |
| "recall": 0.013358778625954198, |
| "f0.5": 0.06118881118881119, |
| "loads_per_turn": 0.02696629213483146, |
| "empty_turn_rate": 0.9730337078651685 |
| } |
| ] |
| } |
| }, |
| "B5-094": { |
| "original_report_sha256": "35c09c36938053b2eb285f4ca8700e2b61b0bb2ba96f1a63d1bbe2004e929bcf", |
| "input_sha256": { |
| "predictions": "e70357d58524a0714882aef5c5de98a6c4cf74a3ad82ae23ee1ed9f97789d1c0", |
| "rows": "67c4c86e3385f0241048f2633d97a4aad78a53a3b7a540592937a20eba669fd3", |
| "corpus": "c77abd8b40965c05cc43a7545e8c2db2405bdcfc18a73bad254632d523b67bcd" |
| }, |
| "prediction_metadata_sha256": "de8a6974d1ca4a615f319089b2a6b023e5e57ede9bc03e52de5aa9524bde11c9", |
| "encoding": "independent", |
| "tool_option_words": 60, |
| "head_sha256": "550f30b94d5c6a8680d2cb2a607ebc9ee33e709c7679f72606d1e9fda17f2e90", |
| "truth": { |
| "consensus": [ |
| { |
| "threshold": 0.95, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 137, |
| "empty": 338, |
| "needed_turns": 216, |
| "any_needed_loaded": 77, |
| "all_needed_loaded": 53, |
| "tn": 29780, |
| "fp": 18, |
| "fn": 202, |
| "unknown_loaded": 36, |
| "tp": 83 |
| }, |
| "precision": 0.8217821782178217, |
| "recall": 0.2912280701754386, |
| "f0.5": 0.602322206095791, |
| "loads_per_turn": 0.30786516853932583, |
| "empty_turn_rate": 0.7595505617977528 |
| }, |
| { |
| "threshold": 0.94, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 161, |
| "empty": 326, |
| "needed_turns": 216, |
| "any_needed_loaded": 83, |
| "all_needed_loaded": 58, |
| "tn": 29775, |
| "fp": 23, |
| "fn": 194, |
| "unknown_loaded": 47, |
| "tp": 91 |
| }, |
| "precision": 0.7982456140350878, |
| "recall": 0.3192982456140351, |
| "f0.5": 0.6140350877192983, |
| "loads_per_turn": 0.36179775280898874, |
| "empty_turn_rate": 0.7325842696629213 |
| } |
| ], |
| "called": [ |
| { |
| "threshold": 0.95, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 137, |
| "empty": 338, |
| "needed_turns": 228, |
| "any_needed_loaded": 71, |
| "all_needed_loaded": 19, |
| "tn": 30485, |
| "fn": 449, |
| "fp": 62, |
| "tp": 75 |
| }, |
| "precision": 0.5474452554744526, |
| "recall": 0.1431297709923664, |
| "f0.5": 0.3498134328358209, |
| "loads_per_turn": 0.30786516853932583, |
| "empty_turn_rate": 0.7595505617977528 |
| }, |
| { |
| "threshold": 0.94, |
| "counts": { |
| "out_of_catalog_labels": 31, |
| "out_of_catalog_positives": 0, |
| "turns": 445, |
| "loads": 161, |
| "empty": 326, |
| "needed_turns": 228, |
| "any_needed_loaded": 83, |
| "all_needed_loaded": 21, |
| "tn": 30476, |
| "fn": 434, |
| "fp": 71, |
| "tp": 90 |
| }, |
| "precision": 0.5590062111801242, |
| "recall": 0.1717557251908397, |
| "f0.5": 0.3852739726027397, |
| "loads_per_turn": 0.36179775280898874, |
| "empty_turn_rate": 0.7325842696629213 |
| } |
| ] |
| } |
| } |
| } |
| } |
|
|
