| { |
| "n_test": 117, |
| "n_eval": 107, |
| "strata": [ |
| "short", |
| "medium", |
| "long", |
| "very_long" |
| ], |
| "stages": [ |
| "0.25", |
| "0.40", |
| "0.60", |
| "0.75" |
| ], |
| "primary": "0.40", |
| "methods": { |
| "naive_onplan": { |
| "family": "naive", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 4.44, |
| "eac_mpe_med": -1.91, |
| "finish_err_med": 1.0, |
| "finish_ape_med": 7.69, |
| "finish_mpe_med": -7.69 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 4.44, |
| "eac_mpe_med": -1.91, |
| "finish_err_med": 1.0, |
| "finish_ape_med": 7.69, |
| "finish_mpe_med": -7.69 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 4.44, |
| "eac_mpe_med": -1.91, |
| "finish_err_med": 1.0, |
| "finish_ape_med": 7.69, |
| "finish_mpe_med": -7.69 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 4.44, |
| "eac_mpe_med": -1.91, |
| "finish_err_med": 1.0, |
| "finish_ape_med": 7.69, |
| "finish_mpe_med": -7.69 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 1.0, |
| "eac_ape_med": 4.21, |
| "eac_mpe_med": -2.48, |
| "finish_err_med": 0.0, |
| "finish_ape_med": 0.0, |
| "finish_mpe_med": 0.0 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 1.0, |
| "eac_ape_med": 5.21, |
| "eac_mpe_med": -0.67, |
| "finish_err_med": 1.0, |
| "finish_ape_med": 11.11, |
| "finish_mpe_med": -11.11 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.13, |
| "eac_mpe_med": -2.71, |
| "finish_err_med": 1.0, |
| "finish_ape_med": 7.69, |
| "finish_mpe_med": -7.69 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "eac_ape_med": 5.86, |
| "eac_mpe_med": -5.86, |
| "finish_err_med": 6.0, |
| "finish_ape_med": 14.87, |
| "finish_mpe_med": -14.87 |
| } |
| } |
| }, |
| "last_value": { |
| "family": "naive", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 4.22, |
| "eac_mpe_med": -0.1, |
| "finish_err_med": 3.61, |
| "finish_ape_med": 36.88, |
| "finish_mpe_med": 10.43 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 4.22, |
| "eac_mpe_med": -0.58, |
| "finish_err_med": 2.16, |
| "finish_ape_med": 26.31, |
| "finish_mpe_med": 8.05 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.22, |
| "eac_mpe_med": -0.41, |
| "finish_err_med": 2.05, |
| "finish_ape_med": 23.83, |
| "finish_mpe_med": 8.05 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.39, |
| "eac_mpe_med": -0.13, |
| "finish_err_med": 1.19, |
| "finish_ape_med": 15.01, |
| "finish_mpe_med": -2.43 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.22, |
| "eac_mpe_med": 0.06, |
| "finish_err_med": 0.63, |
| "finish_ape_med": 14.57, |
| "finish_mpe_med": 8.05 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.95, |
| "eac_mpe_med": -0.38, |
| "finish_err_med": 2.63, |
| "finish_ape_med": 26.31, |
| "finish_mpe_med": -4.43 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 1.0, |
| "eac_ape_med": 10.25, |
| "eac_mpe_med": -4.78, |
| "finish_err_med": 8.54, |
| "finish_ape_med": 60.97, |
| "finish_mpe_med": 60.97 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "eac_ape_med": 41.88, |
| "eac_mpe_med": -2.49, |
| "finish_err_med": 30.32, |
| "finish_ape_med": 79.88, |
| "finish_mpe_med": 79.88 |
| } |
| } |
| }, |
| "linear": { |
| "family": "naive", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 12.44, |
| "eac_mpe_med": -5.07, |
| "finish_err_med": 5.09, |
| "finish_ape_med": 46.78, |
| "finish_mpe_med": 3.03 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 8.77, |
| "eac_mpe_med": -2.21, |
| "finish_err_med": 3.67, |
| "finish_ape_med": 35.22, |
| "finish_mpe_med": 1.24 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 4.05, |
| "eac_mpe_med": -1.51, |
| "finish_err_med": 2.18, |
| "finish_ape_med": 23.85, |
| "finish_mpe_med": -0.37 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.1, |
| "eac_mpe_med": -0.81, |
| "finish_err_med": 1.6, |
| "finish_ape_med": 18.11, |
| "finish_mpe_med": -8.98 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.56, |
| "eac_mpe_med": -0.81, |
| "finish_err_med": 0.75, |
| "finish_ape_med": 18.16, |
| "finish_mpe_med": 0.83 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 1.0, |
| "eac_ape_med": 24.1, |
| "eac_mpe_med": -3.34, |
| "finish_err_med": 4.22, |
| "finish_ape_med": 45.92, |
| "finish_mpe_med": -17.2 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 1.0, |
| "eac_ape_med": 17.08, |
| "eac_mpe_med": -8.75, |
| "finish_err_med": 6.34, |
| "finish_ape_med": 48.79, |
| "finish_mpe_med": 17.16 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "eac_ape_med": 15.02, |
| "eac_mpe_med": -5.33, |
| "finish_err_med": 40.13, |
| "finish_ape_med": 103.58, |
| "finish_mpe_med": 85.44 |
| } |
| } |
| }, |
| "logistic": { |
| "family": "classical", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 24.2, |
| "eac_mpe_med": -11.66, |
| "finish_err_med": 6.36, |
| "finish_ape_med": 51.73, |
| "finish_mpe_med": 5.24 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 21.3, |
| "eac_mpe_med": -17.39, |
| "finish_err_med": 14.0, |
| "finish_ape_med": 173.17, |
| "finish_mpe_med": 173.17 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 13.32, |
| "eac_mpe_med": -10.53, |
| "finish_err_med": 11.0, |
| "finish_ape_med": 166.67, |
| "finish_mpe_med": 166.67 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 4.1, |
| "eac_mpe_med": -2.1, |
| "finish_err_med": 5.62, |
| "finish_ape_med": 28.03, |
| "finish_mpe_med": 19.23 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.56, |
| "eac_mpe_med": -0.81, |
| "finish_err_med": 0.75, |
| "finish_ape_med": 18.16, |
| "finish_mpe_med": 0.83 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 1.0, |
| "eac_ape_med": 43.86, |
| "eac_mpe_med": -38.12, |
| "finish_err_med": 23.0, |
| "finish_ape_med": 255.56, |
| "finish_mpe_med": 255.56 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 1.0, |
| "eac_ape_med": 57.06, |
| "eac_mpe_med": -57.06, |
| "finish_err_med": 44.0, |
| "finish_ape_med": 263.64, |
| "finish_mpe_med": 263.64 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "eac_ape_med": 50.28, |
| "eac_mpe_med": -50.28, |
| "finish_err_med": 69.0, |
| "finish_ape_med": 170.48, |
| "finish_mpe_med": 170.48 |
| } |
| } |
| }, |
| "growth_curve": { |
| "family": "classical", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.23, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 1.91, |
| "finish_ape_med": 22.0, |
| "finish_mpe_med": 0.62 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.37, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 1.55, |
| "finish_ape_med": 18.65, |
| "finish_mpe_med": 0.0 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 1.54, |
| "eac_mpe_med": -0.0, |
| "finish_err_med": 1.11, |
| "finish_ape_med": 15.09, |
| "finish_mpe_med": 2.0 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 1.28, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 1.2, |
| "finish_ape_med": 13.95, |
| "finish_mpe_med": 1.24 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 1.0, |
| "eac_ape_med": 1.54, |
| "eac_mpe_med": -0.0, |
| "finish_err_med": 1.0, |
| "finish_ape_med": 21.13, |
| "finish_mpe_med": 11.05 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.41, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 1.61, |
| "finish_ape_med": 14.67, |
| "finish_mpe_med": 0.0 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.02, |
| "eac_mpe_med": -0.1, |
| "finish_err_med": 3.39, |
| "finish_ape_med": 17.96, |
| "finish_mpe_med": 0.0 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "eac_ape_med": 6.04, |
| "eac_mpe_med": 0.32, |
| "finish_err_med": 10.35, |
| "finish_ape_med": 20.36, |
| "finish_mpe_med": -6.07 |
| } |
| } |
| }, |
| "exp_smoothing": { |
| "family": "classical", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 11.66, |
| "eac_mpe_med": -3.06, |
| "finish_err_med": 4.69, |
| "finish_ape_med": 46.89, |
| "finish_mpe_med": 2.19 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 7.59, |
| "eac_mpe_med": -1.41, |
| "finish_err_med": 3.69, |
| "finish_ape_med": 36.85, |
| "finish_mpe_med": 0.84 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 4.9, |
| "eac_mpe_med": -1.23, |
| "finish_err_med": 2.17, |
| "finish_ape_med": 20.5, |
| "finish_mpe_med": -2.22 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.24, |
| "eac_mpe_med": -0.7, |
| "finish_err_med": 1.6, |
| "finish_ape_med": 18.09, |
| "finish_mpe_med": -2.28 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.56, |
| "eac_mpe_med": -0.81, |
| "finish_err_med": 0.75, |
| "finish_ape_med": 16.8, |
| "finish_mpe_med": 0.84 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 1.0, |
| "eac_ape_med": 23.67, |
| "eac_mpe_med": -2.18, |
| "finish_err_med": 4.43, |
| "finish_ape_med": 45.9, |
| "finish_mpe_med": -25.25 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 1.0, |
| "eac_ape_med": 11.69, |
| "eac_mpe_med": -2.09, |
| "finish_err_med": 6.44, |
| "finish_ape_med": 44.72, |
| "finish_mpe_med": 12.31 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "eac_ape_med": 52.79, |
| "eac_mpe_med": -1.95, |
| "finish_err_med": 69.0, |
| "finish_ape_med": 170.48, |
| "finish_mpe_med": 170.48 |
| } |
| } |
| }, |
| "earned_schedule": { |
| "family": "classical", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.23, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 0.98, |
| "finish_ape_med": 11.11, |
| "finish_mpe_med": 0.0 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.37, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 1.0, |
| "finish_ape_med": 10.71, |
| "finish_mpe_med": 0.0 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 1.54, |
| "eac_mpe_med": -0.0, |
| "finish_err_med": 0.75, |
| "finish_ape_med": 7.71, |
| "finish_mpe_med": 0.71 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 1.28, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 0.69, |
| "finish_ape_med": 7.1, |
| "finish_mpe_med": 0.0 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 1.0, |
| "eac_ape_med": 1.54, |
| "eac_mpe_med": -0.0, |
| "finish_err_med": 0.35, |
| "finish_ape_med": 8.32, |
| "finish_mpe_med": 0.0 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.41, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 1.0, |
| "finish_ape_med": 11.11, |
| "finish_mpe_med": 0.34 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.02, |
| "eac_mpe_med": -0.1, |
| "finish_err_med": 2.67, |
| "finish_ape_med": 15.33, |
| "finish_mpe_med": 0.0 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "eac_ape_med": 6.04, |
| "eac_mpe_med": 0.32, |
| "finish_err_med": 5.37, |
| "finish_ape_med": 10.39, |
| "finish_mpe_med": 1.5 |
| } |
| } |
| }, |
| "evm_cpi_spi": { |
| "family": "classical", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 10.0, |
| "eac_mpe_med": 4.67, |
| "finish_err_med": 0.98, |
| "finish_ape_med": 11.11, |
| "finish_mpe_med": 0.0 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 9.65, |
| "eac_mpe_med": 4.61, |
| "finish_err_med": 1.0, |
| "finish_ape_med": 10.71, |
| "finish_mpe_med": 0.0 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 4.29, |
| "eac_mpe_med": 2.61, |
| "finish_err_med": 0.75, |
| "finish_ape_med": 7.71, |
| "finish_mpe_med": 0.71 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.23, |
| "eac_mpe_med": 1.28, |
| "finish_err_med": 0.69, |
| "finish_ape_med": 7.1, |
| "finish_mpe_med": 0.0 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.7, |
| "eac_mpe_med": 3.23, |
| "finish_err_med": 0.35, |
| "finish_ape_med": 8.32, |
| "finish_mpe_med": 0.0 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 1.0, |
| "eac_ape_med": 12.71, |
| "eac_mpe_med": 11.65, |
| "finish_err_med": 1.0, |
| "finish_ape_med": 11.11, |
| "finish_mpe_med": 0.34 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 1.0, |
| "eac_ape_med": 15.12, |
| "eac_mpe_med": 4.51, |
| "finish_err_med": 2.67, |
| "finish_ape_med": 15.33, |
| "finish_mpe_med": 0.0 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "eac_ape_med": 22.86, |
| "eac_mpe_med": 15.25, |
| "finish_err_med": 5.37, |
| "finish_ape_med": 10.39, |
| "finish_mpe_med": 1.5 |
| } |
| } |
| }, |
| "xsm": { |
| "family": "classical", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.91, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 1.01, |
| "finish_ape_med": 13.88, |
| "finish_mpe_med": 3.36 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.8, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 0.9, |
| "finish_ape_med": 12.44, |
| "finish_mpe_med": 3.81 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 1.87, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 0.87, |
| "finish_ape_med": 10.02, |
| "finish_mpe_med": 1.12 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 1.48, |
| "eac_mpe_med": -0.0, |
| "finish_err_med": 0.7, |
| "finish_ape_med": 6.9, |
| "finish_mpe_med": 0.18 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 1.0, |
| "eac_ape_med": 1.56, |
| "eac_mpe_med": 0.44, |
| "finish_err_med": 0.34, |
| "finish_ape_med": 6.9, |
| "finish_mpe_med": 0.0 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.91, |
| "eac_mpe_med": 0.31, |
| "finish_err_med": 1.26, |
| "finish_ape_med": 12.68, |
| "finish_mpe_med": 5.41 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 1.0, |
| "eac_ape_med": 4.16, |
| "eac_mpe_med": -0.23, |
| "finish_err_med": 4.24, |
| "finish_ape_med": 23.45, |
| "finish_mpe_med": 12.29 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "eac_ape_med": 14.87, |
| "eac_mpe_med": 0.32, |
| "finish_err_med": 7.64, |
| "finish_ape_med": 17.33, |
| "finish_mpe_med": 10.07 |
| } |
| } |
| }, |
| "timesfm_2.5": { |
| "family": "foundation", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 4.39, |
| "eac_mpe_med": -0.1, |
| "finish_err_med": 2.45, |
| "finish_ape_med": 27.2, |
| "finish_mpe_med": 11.03, |
| "coverage_med": 1.0 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 4.04, |
| "eac_mpe_med": -0.39, |
| "finish_err_med": 2.19, |
| "finish_ape_med": 24.15, |
| "finish_mpe_med": 11.6, |
| "coverage_med": 0.923 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.29, |
| "eac_mpe_med": -0.1, |
| "finish_err_med": 1.47, |
| "finish_ape_med": 18.3, |
| "finish_mpe_med": 4.93, |
| "coverage_med": 1.0 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.24, |
| "eac_mpe_med": 0.06, |
| "finish_err_med": 1.21, |
| "finish_ape_med": 14.44, |
| "finish_mpe_med": -3.11, |
| "coverage_med": 1.0 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.99, |
| "eac_mpe_med": 0.14, |
| "finish_err_med": 0.59, |
| "finish_ape_med": 13.51, |
| "finish_mpe_med": 9.12, |
| "coverage_med": 0.667 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.52, |
| "eac_mpe_med": -0.1, |
| "finish_err_med": 2.79, |
| "finish_ape_med": 27.95, |
| "finish_mpe_med": 2.11, |
| "coverage_med": 1.0 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 1.0, |
| "eac_ape_med": 4.29, |
| "eac_mpe_med": -2.55, |
| "finish_err_med": 6.65, |
| "finish_ape_med": 41.24, |
| "finish_mpe_med": 41.24, |
| "coverage_med": 0.909 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "eac_ape_med": 22.45, |
| "eac_mpe_med": -3.93, |
| "finish_err_med": 12.45, |
| "finish_ape_med": 41.97, |
| "finish_mpe_med": 39.78, |
| "coverage_med": 0.921 |
| } |
| } |
| }, |
| "chronos_2": { |
| "family": "foundation", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 4.76, |
| "eac_mpe_med": -0.1, |
| "finish_err_med": 3.22, |
| "finish_ape_med": 33.9, |
| "finish_mpe_med": 10.09, |
| "coverage_med": 1.0 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.93, |
| "eac_mpe_med": -0.12, |
| "finish_err_med": 2.13, |
| "finish_ape_med": 22.38, |
| "finish_mpe_med": 7.26, |
| "coverage_med": 1.0 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.18, |
| "eac_mpe_med": -0.1, |
| "finish_err_med": 1.61, |
| "finish_ape_med": 18.51, |
| "finish_mpe_med": 7.03, |
| "coverage_med": 1.0 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.24, |
| "eac_mpe_med": -0.1, |
| "finish_err_med": 1.28, |
| "finish_ape_med": 13.55, |
| "finish_mpe_med": -0.18, |
| "coverage_med": 1.0 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.74, |
| "eac_mpe_med": 0.61, |
| "finish_err_med": 0.71, |
| "finish_ape_med": 15.43, |
| "finish_mpe_med": 8.55, |
| "coverage_med": 1.0 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.93, |
| "eac_mpe_med": -0.18, |
| "finish_err_med": 2.35, |
| "finish_ape_med": 20.41, |
| "finish_mpe_med": -2.33, |
| "coverage_med": 1.0 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.52, |
| "eac_mpe_med": -1.17, |
| "finish_err_med": 5.43, |
| "finish_ape_med": 38.5, |
| "finish_mpe_med": 32.68, |
| "coverage_med": 1.0 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "eac_ape_med": 14.55, |
| "eac_mpe_med": -0.51, |
| "finish_err_med": 19.78, |
| "finish_ape_med": 46.02, |
| "finish_mpe_med": 22.7, |
| "coverage_med": 0.98 |
| } |
| } |
| }, |
| "chronos_2_cov": { |
| "family": "foundation", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 5.34, |
| "eac_mpe_med": -0.22, |
| "finish_err_med": 3.05, |
| "finish_ape_med": 33.9, |
| "finish_mpe_med": 10.7, |
| "coverage_med": 1.0 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 4.43, |
| "eac_mpe_med": -0.1, |
| "finish_err_med": 2.29, |
| "finish_ape_med": 22.11, |
| "finish_mpe_med": 9.01, |
| "coverage_med": 1.0 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.82, |
| "eac_mpe_med": -0.26, |
| "finish_err_med": 1.4, |
| "finish_ape_med": 18.53, |
| "finish_mpe_med": 7.16, |
| "coverage_med": 1.0 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.08, |
| "eac_mpe_med": -0.1, |
| "finish_err_med": 1.26, |
| "finish_ape_med": 14.64, |
| "finish_mpe_med": 0.17, |
| "coverage_med": 1.0 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 1.0, |
| "eac_ape_med": 4.04, |
| "eac_mpe_med": 0.63, |
| "finish_err_med": 0.66, |
| "finish_ape_med": 13.57, |
| "finish_mpe_med": 8.97, |
| "coverage_med": 1.0 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 1.0, |
| "eac_ape_med": 4.03, |
| "eac_mpe_med": -0.1, |
| "finish_err_med": 2.32, |
| "finish_ape_med": 24.07, |
| "finish_mpe_med": -0.08, |
| "coverage_med": 1.0 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 1.0, |
| "eac_ape_med": 5.26, |
| "eac_mpe_med": -1.72, |
| "finish_err_med": 5.58, |
| "finish_ape_med": 39.06, |
| "finish_mpe_med": 37.9, |
| "coverage_med": 1.0 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "eac_ape_med": 13.75, |
| "eac_mpe_med": -1.18, |
| "finish_err_med": 9.18, |
| "finish_ape_med": 31.2, |
| "finish_mpe_med": 21.12, |
| "coverage_med": 1.0 |
| } |
| } |
| }, |
| "naive_dist": { |
| "family": "control", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.32, |
| "eac_mpe_med": 0.05, |
| "finish_err_med": 0.71, |
| "finish_ape_med": 14.29, |
| "finish_mpe_med": 5.49 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.32, |
| "eac_mpe_med": 0.05, |
| "finish_err_med": 0.71, |
| "finish_ape_med": 14.29, |
| "finish_mpe_med": 5.49 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.32, |
| "eac_mpe_med": 0.05, |
| "finish_err_med": 0.71, |
| "finish_ape_med": 14.29, |
| "finish_mpe_med": 5.49 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.32, |
| "eac_mpe_med": 0.05, |
| "finish_err_med": 0.71, |
| "finish_ape_med": 14.29, |
| "finish_mpe_med": 5.49 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.53, |
| "eac_mpe_med": -0.53, |
| "finish_err_med": 0.57, |
| "finish_ape_med": 14.29, |
| "finish_mpe_med": 14.29 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 1.0, |
| "eac_ape_med": 5.18, |
| "eac_mpe_med": 1.32, |
| "finish_err_med": 0.86, |
| "finish_ape_med": 8.57, |
| "finish_mpe_med": 1.59 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.6, |
| "eac_mpe_med": -0.77, |
| "finish_err_med": 1.43, |
| "finish_ape_med": 7.94, |
| "finish_mpe_med": 5.49 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.97, |
| "eac_mpe_med": -3.97, |
| "finish_err_med": 3.93, |
| "finish_ape_med": 12.9, |
| "finish_mpe_med": -2.71 |
| } |
| } |
| }, |
| "ml_ridge": { |
| "family": "ml", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 0.9906542056074766, |
| "eac_ape_med": 4.39, |
| "eac_mpe_med": 1.61, |
| "finish_err_med": 0.98, |
| "finish_ape_med": 12.31, |
| "finish_mpe_med": 8.94 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.67, |
| "eac_mpe_med": 0.97, |
| "finish_err_med": 1.12, |
| "finish_ape_med": 11.15, |
| "finish_mpe_med": 9.1 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.42, |
| "eac_mpe_med": 1.08, |
| "finish_err_med": 0.52, |
| "finish_ape_med": 5.3, |
| "finish_mpe_med": 2.91 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.02, |
| "eac_mpe_med": 1.17, |
| "finish_err_med": 0.3, |
| "finish_ape_med": 3.8, |
| "finish_mpe_med": 3.37 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.74, |
| "eac_mpe_med": 1.34, |
| "finish_err_med": 1.12, |
| "finish_ape_med": 27.73, |
| "finish_mpe_med": 27.73 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 1.0, |
| "eac_ape_med": 5.09, |
| "eac_mpe_med": 1.63, |
| "finish_err_med": 0.46, |
| "finish_ape_med": 4.34, |
| "finish_mpe_med": -2.51 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 1.0, |
| "eac_ape_med": 4.32, |
| "eac_mpe_med": -0.88, |
| "finish_err_med": 1.47, |
| "finish_ape_med": 7.98, |
| "finish_mpe_med": 2.97 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "eac_ape_med": 7.02, |
| "eac_mpe_med": 1.05, |
| "finish_err_med": 3.79, |
| "finish_ape_med": 8.97, |
| "finish_mpe_med": 3.18 |
| } |
| } |
| }, |
| "ml_svr": { |
| "family": "ml", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 5.05, |
| "eac_mpe_med": 2.0, |
| "finish_err_med": 1.3, |
| "finish_ape_med": 16.21, |
| "finish_mpe_med": 13.89 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.61, |
| "eac_mpe_med": 1.0, |
| "finish_err_med": 1.25, |
| "finish_ape_med": 14.04, |
| "finish_mpe_med": 9.1 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.45, |
| "eac_mpe_med": 0.7, |
| "finish_err_med": 0.93, |
| "finish_ape_med": 8.66, |
| "finish_mpe_med": 6.01 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.27, |
| "eac_mpe_med": 0.58, |
| "finish_err_med": 0.89, |
| "finish_ape_med": 9.89, |
| "finish_mpe_med": 8.61 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.23, |
| "eac_mpe_med": 1.66, |
| "finish_err_med": 1.23, |
| "finish_ape_med": 30.61, |
| "finish_mpe_med": 25.3 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 1.0, |
| "eac_ape_med": 4.17, |
| "eac_mpe_med": 1.33, |
| "finish_err_med": 0.91, |
| "finish_ape_med": 9.08, |
| "finish_mpe_med": 1.15 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 1.0, |
| "eac_ape_med": 4.16, |
| "eac_mpe_med": -0.21, |
| "finish_err_med": 1.28, |
| "finish_ape_med": 6.38, |
| "finish_mpe_med": -1.4 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "eac_ape_med": 8.58, |
| "eac_mpe_med": 2.05, |
| "finish_err_med": 1.5, |
| "finish_ape_med": 4.06, |
| "finish_mpe_med": 2.06 |
| } |
| } |
| }, |
| "ml_rf": { |
| "family": "ml", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.84, |
| "eac_mpe_med": 0.57, |
| "finish_err_med": 0.97, |
| "finish_ape_med": 10.99, |
| "finish_mpe_med": 8.23 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.26, |
| "eac_mpe_med": -0.03, |
| "finish_err_med": 0.83, |
| "finish_ape_med": 6.09, |
| "finish_mpe_med": 3.78 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 1.53, |
| "eac_mpe_med": -0.08, |
| "finish_err_med": 0.36, |
| "finish_ape_med": 3.61, |
| "finish_mpe_med": 2.47 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 1.49, |
| "eac_mpe_med": 0.05, |
| "finish_err_med": 0.03, |
| "finish_ape_med": 0.24, |
| "finish_mpe_med": 0.0 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 1.0, |
| "eac_ape_med": 1.64, |
| "eac_mpe_med": 0.4, |
| "finish_err_med": 0.89, |
| "finish_ape_med": 21.06, |
| "finish_mpe_med": 21.06 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.8, |
| "eac_mpe_med": 0.98, |
| "finish_err_med": 0.37, |
| "finish_ape_med": 3.69, |
| "finish_mpe_med": 0.0 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.6, |
| "eac_mpe_med": -0.55, |
| "finish_err_med": 0.73, |
| "finish_ape_med": 4.29, |
| "finish_mpe_med": 0.46 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "eac_ape_med": 6.29, |
| "eac_mpe_med": -0.19, |
| "finish_err_med": 0.89, |
| "finish_ape_med": 2.83, |
| "finish_mpe_med": -1.19 |
| } |
| } |
| }, |
| "ml_histgbm": { |
| "family": "ml", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.7, |
| "eac_mpe_med": 0.47, |
| "finish_err_med": 0.98, |
| "finish_ape_med": 11.14, |
| "finish_mpe_med": 8.13 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.9, |
| "eac_mpe_med": 0.02, |
| "finish_err_med": 0.94, |
| "finish_ape_med": 6.96, |
| "finish_mpe_med": 5.13 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 1.81, |
| "eac_mpe_med": -0.02, |
| "finish_err_med": 0.35, |
| "finish_ape_med": 3.7, |
| "finish_mpe_med": 2.3 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 1.19, |
| "eac_mpe_med": 0.17, |
| "finish_err_med": 0.13, |
| "finish_ape_med": 1.67, |
| "finish_mpe_med": 0.88 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 1.0, |
| "eac_ape_med": 1.72, |
| "eac_mpe_med": 0.63, |
| "finish_err_med": 0.99, |
| "finish_ape_med": 24.18, |
| "finish_mpe_med": 23.59 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.0, |
| "eac_mpe_med": 0.46, |
| "finish_err_med": 0.29, |
| "finish_ape_med": 4.2, |
| "finish_mpe_med": -0.02 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.9, |
| "eac_mpe_med": -1.15, |
| "finish_err_med": 0.84, |
| "finish_ape_med": 5.3, |
| "finish_mpe_med": 1.51 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "eac_ape_med": 7.0, |
| "eac_mpe_med": 0.66, |
| "finish_err_med": 1.22, |
| "finish_ape_med": 3.26, |
| "finish_mpe_med": -1.6 |
| } |
| } |
| }, |
| "ml_xgboost": { |
| "family": "ml", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.6, |
| "eac_mpe_med": 0.26, |
| "finish_err_med": 0.98, |
| "finish_ape_med": 9.83, |
| "finish_mpe_med": 6.91 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.31, |
| "eac_mpe_med": 0.19, |
| "finish_err_med": 0.93, |
| "finish_ape_med": 7.67, |
| "finish_mpe_med": 5.53 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 1.69, |
| "eac_mpe_med": -0.21, |
| "finish_err_med": 0.45, |
| "finish_ape_med": 3.87, |
| "finish_mpe_med": 2.92 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 1.38, |
| "eac_mpe_med": 0.16, |
| "finish_err_med": 0.2, |
| "finish_ape_med": 2.65, |
| "finish_mpe_med": 1.67 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 1.0, |
| "eac_ape_med": 1.63, |
| "eac_mpe_med": 0.53, |
| "finish_err_med": 0.97, |
| "finish_ape_med": 23.64, |
| "finish_mpe_med": 23.63 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 1.0, |
| "eac_ape_med": 4.05, |
| "eac_mpe_med": 0.43, |
| "finish_err_med": 0.41, |
| "finish_ape_med": 4.13, |
| "finish_mpe_med": 1.3 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.04, |
| "eac_mpe_med": -1.05, |
| "finish_err_med": 0.82, |
| "finish_ape_med": 4.04, |
| "finish_mpe_med": 0.43 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "eac_ape_med": 9.22, |
| "eac_mpe_med": -0.07, |
| "finish_err_med": 0.94, |
| "finish_ape_med": 2.71, |
| "finish_mpe_med": -1.04 |
| } |
| } |
| }, |
| "ml_lightgbm": { |
| "family": "ml", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.38, |
| "eac_mpe_med": 0.55, |
| "finish_err_med": 0.99, |
| "finish_ape_med": 10.34, |
| "finish_mpe_med": 8.41 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.92, |
| "eac_mpe_med": 0.2, |
| "finish_err_med": 0.92, |
| "finish_ape_med": 6.41, |
| "finish_mpe_med": 3.98 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 1.9, |
| "eac_mpe_med": -0.1, |
| "finish_err_med": 0.4, |
| "finish_ape_med": 3.87, |
| "finish_mpe_med": 2.36 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 1.19, |
| "eac_mpe_med": 0.16, |
| "finish_err_med": 0.16, |
| "finish_ape_med": 1.68, |
| "finish_mpe_med": 1.0 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.16, |
| "eac_mpe_med": 0.46, |
| "finish_err_med": 0.98, |
| "finish_ape_med": 23.92, |
| "finish_mpe_med": 23.38 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 1.0, |
| "eac_ape_med": 4.05, |
| "eac_mpe_med": 0.44, |
| "finish_err_med": 0.23, |
| "finish_ape_med": 2.92, |
| "finish_mpe_med": 0.53 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.84, |
| "eac_mpe_med": -0.9, |
| "finish_err_med": 0.66, |
| "finish_ape_med": 3.82, |
| "finish_mpe_med": 0.74 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "eac_ape_med": 6.42, |
| "eac_mpe_med": 0.07, |
| "finish_err_med": 1.34, |
| "finish_ape_med": 3.15, |
| "finish_mpe_med": 0.18 |
| } |
| } |
| }, |
| "ml_catboost": { |
| "family": "ml", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.63, |
| "eac_mpe_med": 0.82, |
| "finish_err_med": 0.97, |
| "finish_ape_med": 10.03, |
| "finish_mpe_med": 6.64 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.07, |
| "eac_mpe_med": 0.6, |
| "finish_err_med": 0.91, |
| "finish_ape_med": 7.39, |
| "finish_mpe_med": 4.34 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.0, |
| "eac_mpe_med": -0.09, |
| "finish_err_med": 0.41, |
| "finish_ape_med": 4.1, |
| "finish_mpe_med": 2.86 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 1.37, |
| "eac_mpe_med": -0.0, |
| "finish_err_med": 0.18, |
| "finish_ape_med": 2.1, |
| "finish_mpe_med": 1.57 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 1.0, |
| "eac_ape_med": 1.92, |
| "eac_mpe_med": 0.88, |
| "finish_err_med": 0.95, |
| "finish_ape_med": 23.27, |
| "finish_mpe_med": 23.22 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.94, |
| "eac_mpe_med": 0.6, |
| "finish_err_med": 0.29, |
| "finish_ape_med": 3.21, |
| "finish_mpe_med": 0.77 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.85, |
| "eac_mpe_med": -0.24, |
| "finish_err_med": 0.61, |
| "finish_ape_med": 4.25, |
| "finish_mpe_med": -0.38 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "eac_ape_med": 9.53, |
| "eac_mpe_med": 0.97, |
| "finish_err_med": 1.31, |
| "finish_ape_med": 3.22, |
| "finish_mpe_med": -2.04 |
| } |
| } |
| }, |
| "ml_mlp": { |
| "family": "ml", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 0.9906542056074766, |
| "eac_ape_med": 5.8, |
| "eac_mpe_med": 2.35, |
| "finish_err_med": 1.31, |
| "finish_ape_med": 13.59, |
| "finish_mpe_med": 11.09 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 5.85, |
| "eac_mpe_med": 2.65, |
| "finish_err_med": 1.04, |
| "finish_ape_med": 12.16, |
| "finish_mpe_med": 8.06 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 0.9906542056074766, |
| "eac_ape_med": 3.89, |
| "eac_mpe_med": 1.09, |
| "finish_err_med": 0.69, |
| "finish_ape_med": 8.32, |
| "finish_mpe_med": 5.21 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 4.49, |
| "eac_mpe_med": 1.69, |
| "finish_err_med": 0.81, |
| "finish_ape_med": 7.67, |
| "finish_mpe_med": 7.31 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 1.0, |
| "eac_ape_med": 5.85, |
| "eac_mpe_med": 3.88, |
| "finish_err_med": 1.07, |
| "finish_ape_med": 26.08, |
| "finish_mpe_med": 26.08 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 1.0, |
| "eac_ape_med": 5.65, |
| "eac_mpe_med": 4.21, |
| "finish_err_med": 0.7, |
| "finish_ape_med": 7.4, |
| "finish_mpe_med": 3.23 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 1.0, |
| "eac_ape_med": 5.97, |
| "eac_mpe_med": -0.77, |
| "finish_err_med": 1.05, |
| "finish_ape_med": 5.55, |
| "finish_mpe_med": 1.46 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "eac_ape_med": 8.93, |
| "eac_mpe_med": 0.02, |
| "finish_err_med": 1.5, |
| "finish_ape_med": 4.24, |
| "finish_mpe_med": -2.72 |
| } |
| } |
| }, |
| "ml_tabpfn": { |
| "family": "ml", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.73, |
| "eac_mpe_med": 0.6, |
| "finish_err_med": 0.97, |
| "finish_ape_med": 8.28, |
| "finish_mpe_med": 5.88 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.0, |
| "eac_mpe_med": 0.07, |
| "finish_err_med": 0.93, |
| "finish_ape_med": 5.77, |
| "finish_mpe_med": 3.55 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 1.77, |
| "eac_mpe_med": 0.07, |
| "finish_err_med": 0.41, |
| "finish_ape_med": 2.43, |
| "finish_mpe_med": 0.57 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 1.54, |
| "eac_mpe_med": 0.02, |
| "finish_err_med": 0.04, |
| "finish_ape_med": 0.19, |
| "finish_mpe_med": 0.01 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 1.0, |
| "eac_ape_med": 1.68, |
| "eac_mpe_med": 0.17, |
| "finish_err_med": 0.98, |
| "finish_ape_med": 24.2, |
| "finish_mpe_med": 24.2 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.86, |
| "eac_mpe_med": 0.4, |
| "finish_err_med": 0.46, |
| "finish_ape_med": 4.6, |
| "finish_mpe_med": -0.01 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.48, |
| "eac_mpe_med": -0.39, |
| "finish_err_med": 0.72, |
| "finish_ape_med": 4.62, |
| "finish_mpe_med": 0.17 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "eac_ape_med": 6.75, |
| "eac_mpe_med": 0.52, |
| "finish_err_med": 0.89, |
| "finish_ape_med": 2.98, |
| "finish_mpe_med": -1.0 |
| } |
| } |
| }, |
| "ml_lightgbm_quantile": { |
| "family": "ml", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.99, |
| "eac_mpe_med": 0.39, |
| "finish_err_med": 1.0, |
| "finish_ape_med": 9.15, |
| "finish_mpe_med": 4.78, |
| "coverage_med": 0.411, |
| "finish_cov_med": 0.299 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.6, |
| "eac_mpe_med": 0.38, |
| "finish_err_med": 0.98, |
| "finish_ape_med": 7.14, |
| "finish_mpe_med": 1.91, |
| "coverage_med": 0.374, |
| "finish_cov_med": 0.336 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 1.61, |
| "eac_mpe_med": -0.14, |
| "finish_err_med": 0.58, |
| "finish_ape_med": 3.37, |
| "finish_mpe_med": 1.59, |
| "coverage_med": 0.439, |
| "finish_cov_med": 0.439 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 1.26, |
| "eac_mpe_med": 0.02, |
| "finish_err_med": 0.27, |
| "finish_ape_med": 1.68, |
| "finish_mpe_med": 0.75, |
| "coverage_med": 0.467, |
| "finish_cov_med": 0.327 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 1.0, |
| "eac_ape_med": 1.94, |
| "eac_mpe_med": 0.38, |
| "finish_err_med": 0.99, |
| "finish_ape_med": 24.57, |
| "finish_mpe_med": 23.96, |
| "coverage_med": 0.372, |
| "finish_cov_med": 0.186 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.84, |
| "eac_mpe_med": 0.59, |
| "finish_err_med": 0.61, |
| "finish_ape_med": 7.14, |
| "finish_mpe_med": 0.18, |
| "coverage_med": 0.407, |
| "finish_cov_med": 0.556 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.49, |
| "eac_mpe_med": -0.07, |
| "finish_err_med": 0.68, |
| "finish_ape_med": 2.97, |
| "finish_mpe_med": 0.4, |
| "coverage_med": 0.4, |
| "finish_cov_med": 0.4 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "eac_ape_med": 6.24, |
| "eac_mpe_med": 1.08, |
| "finish_err_med": 1.05, |
| "finish_ape_med": 2.97, |
| "finish_mpe_med": -2.1, |
| "coverage_med": 0.25, |
| "finish_cov_med": 0.25 |
| } |
| } |
| }, |
| "ml_xgboost_realcv": { |
| "family": "ml", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 4.2, |
| "eac_mpe_med": 0.37, |
| "finish_err_med": 0.79, |
| "finish_ape_med": 7.82, |
| "finish_mpe_med": 0.96 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.76, |
| "eac_mpe_med": -0.11, |
| "finish_err_med": 0.61, |
| "finish_ape_med": 5.8, |
| "finish_mpe_med": 0.67 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.66, |
| "eac_mpe_med": -0.32, |
| "finish_err_med": 0.39, |
| "finish_ape_med": 4.49, |
| "finish_mpe_med": 0.33 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.22, |
| "eac_mpe_med": 0.09, |
| "finish_err_med": 0.23, |
| "finish_ape_med": 2.81, |
| "finish_mpe_med": 0.04 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.33, |
| "eac_mpe_med": -0.16, |
| "finish_err_med": 0.3, |
| "finish_ape_med": 6.58, |
| "finish_mpe_med": 1.22 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 1.0, |
| "eac_ape_med": 6.38, |
| "eac_mpe_med": -0.7, |
| "finish_err_med": 0.42, |
| "finish_ape_med": 5.21, |
| "finish_mpe_med": -2.02 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 1.0, |
| "eac_ape_med": 5.4, |
| "eac_mpe_med": 0.78, |
| "finish_err_med": 0.85, |
| "finish_ape_med": 4.89, |
| "finish_mpe_med": 3.58 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "eac_ape_med": 7.88, |
| "eac_mpe_med": -3.33, |
| "finish_err_med": 3.47, |
| "finish_ape_med": 9.13, |
| "finish_mpe_med": -3.73 |
| } |
| } |
| }, |
| "ml_tabpfn_realcv": { |
| "family": "ml", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.72, |
| "eac_mpe_med": 0.06, |
| "finish_err_med": 0.66, |
| "finish_ape_med": 6.25, |
| "finish_mpe_med": 1.01 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.82, |
| "eac_mpe_med": 0.24, |
| "finish_err_med": 0.58, |
| "finish_ape_med": 4.54, |
| "finish_mpe_med": 0.64 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 1.68, |
| "eac_mpe_med": -0.03, |
| "finish_err_med": 0.24, |
| "finish_ape_med": 1.95, |
| "finish_mpe_med": 0.39 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 1.4, |
| "eac_mpe_med": -0.04, |
| "finish_err_med": 0.15, |
| "finish_ape_med": 1.1, |
| "finish_mpe_med": 0.05 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 1.0, |
| "eac_ape_med": 1.4, |
| "eac_mpe_med": -0.27, |
| "finish_err_med": 0.25, |
| "finish_ape_med": 5.82, |
| "finish_mpe_med": 1.06 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 1.0, |
| "eac_ape_med": 5.06, |
| "eac_mpe_med": 0.87, |
| "finish_err_med": 0.59, |
| "finish_ape_med": 5.53, |
| "finish_mpe_med": -0.51 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 1.0, |
| "eac_ape_med": 4.33, |
| "eac_mpe_med": 1.42, |
| "finish_err_med": 0.64, |
| "finish_ape_med": 4.32, |
| "finish_mpe_med": 1.93 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "eac_ape_med": 4.65, |
| "eac_mpe_med": -1.59, |
| "finish_err_med": 1.25, |
| "finish_ape_med": 3.31, |
| "finish_mpe_med": -0.63 |
| } |
| } |
| }, |
| "naive_dist_realcv": { |
| "family": "control", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.55, |
| "eac_mpe_med": -0.13, |
| "finish_err_med": 0.71, |
| "finish_ape_med": 10.0, |
| "finish_mpe_med": -1.0 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.55, |
| "eac_mpe_med": -0.13, |
| "finish_err_med": 0.71, |
| "finish_ape_med": 10.0, |
| "finish_mpe_med": -1.0 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.55, |
| "eac_mpe_med": -0.13, |
| "finish_err_med": 0.71, |
| "finish_ape_med": 10.0, |
| "finish_mpe_med": -1.0 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.55, |
| "eac_mpe_med": -0.13, |
| "finish_err_med": 0.71, |
| "finish_ape_med": 10.0, |
| "finish_mpe_med": -1.0 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 1.0, |
| "eac_ape_med": 2.74, |
| "eac_mpe_med": -0.13, |
| "finish_err_med": 0.44, |
| "finish_ape_med": 10.0, |
| "finish_mpe_med": 6.79 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 1.0, |
| "eac_ape_med": 5.0, |
| "eac_mpe_med": 1.46, |
| "finish_err_med": 0.71, |
| "finish_ape_med": 7.79, |
| "finish_mpe_med": -4.18 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 1.0, |
| "eac_ape_med": 3.91, |
| "eac_mpe_med": -1.22, |
| "finish_err_med": 1.25, |
| "finish_ape_med": 7.93, |
| "finish_mpe_med": -2.26 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "eac_ape_med": 4.02, |
| "eac_mpe_med": -4.02, |
| "finish_err_med": 4.0, |
| "finish_ape_med": 10.81, |
| "finish_mpe_med": -8.65 |
| } |
| } |
| }, |
| "agent_deepseek_v4_flash": { |
| "family": "agent", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "turns_med": 3.0, |
| "err_total": 3, |
| "cost_total": 0.0, |
| "tokens_total": 1685971, |
| "eac_ape_med": 3.15, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 0.67, |
| "finish_mpe_med": 0.0 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "turns_med": 4.0, |
| "err_total": 7, |
| "cost_total": 0.0, |
| "tokens_total": 1848021, |
| "eac_ape_med": 2.4, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 0.6, |
| "finish_mpe_med": 0.0 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "turns_med": 4.0, |
| "err_total": 5, |
| "cost_total": 0.0, |
| "tokens_total": 1974062, |
| "eac_ape_med": 1.61, |
| "eac_mpe_med": -0.0, |
| "finish_err_med": 0.35, |
| "finish_mpe_med": 0.0 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "turns_med": 4.0, |
| "err_total": 7, |
| "cost_total": 0.0, |
| "tokens_total": 1978877, |
| "eac_ape_med": 1.26, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 0.4, |
| "finish_mpe_med": 0.0 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 1.0, |
| "turns_med": 4.0, |
| "err_total": 2, |
| "cost_total": 0.0, |
| "tokens_total": 646885, |
| "eac_ape_med": 1.54, |
| "eac_mpe_med": -0.0, |
| "finish_err_med": 0.35, |
| "finish_mpe_med": 0.0 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 1.0, |
| "turns_med": 3.0, |
| "err_total": 2, |
| "cost_total": 0.0, |
| "tokens_total": 448128, |
| "eac_ape_med": 3.31, |
| "eac_mpe_med": 0.92, |
| "finish_err_med": 0.5, |
| "finish_mpe_med": 0.0 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 1.0, |
| "turns_med": 4.0, |
| "err_total": 3, |
| "cost_total": 0.0, |
| "tokens_total": 491823, |
| "eac_ape_med": 2.11, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 0.84, |
| "finish_mpe_med": 0.0 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "turns_med": 4.0, |
| "err_total": 0, |
| "cost_total": 0.0, |
| "tokens_total": 261185, |
| "eac_ape_med": 7.09, |
| "eac_mpe_med": 0.32, |
| "finish_err_med": 1.3, |
| "finish_mpe_med": -2.04 |
| } |
| } |
| }, |
| "student_MiniCPM5-1B_base": { |
| "family": "agent", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 0.009, |
| "turns_med": 6.0, |
| "err_total": 289, |
| "cost_total": 0.0, |
| "tokens_total": 1209134, |
| "eac_ape_med": 3.36, |
| "eac_mpe_med": -3.36, |
| "finish_err_med": 1797870.0, |
| "finish_mpe_med": 44946750.0 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 0.019, |
| "turns_med": 6.0, |
| "err_total": 167, |
| "cost_total": 0.0, |
| "tokens_total": 1228476, |
| "eac_ape_med": 50.54, |
| "eac_mpe_med": -50.54, |
| "finish_err_med": 1.27, |
| "finish_mpe_med": -14.11 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 0.009, |
| "turns_med": 6.0, |
| "err_total": 232, |
| "cost_total": 0.0, |
| "tokens_total": 1224221, |
| "eac_ape_med": 14.34, |
| "eac_mpe_med": 14.34, |
| "finish_err_med": 30.0, |
| "finish_mpe_med": -96.77 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 0.019, |
| "turns_med": 6.0, |
| "err_total": 316, |
| "cost_total": 0.0, |
| "tokens_total": 1186720, |
| "eac_ape_med": 1.25, |
| "eac_mpe_med": -0.48, |
| "finish_err_med": 32308.0, |
| "finish_mpe_med": 538440.0 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 0.0, |
| "turns_med": 6.0, |
| "err_total": 75, |
| "cost_total": 0.0, |
| "tokens_total": 505928 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 0.037, |
| "turns_med": 6.0, |
| "err_total": 64, |
| "cost_total": 0.0, |
| "tokens_total": 278177, |
| "eac_ape_med": 100.0, |
| "eac_mpe_med": -100.0, |
| "finish_err_med": 2.54, |
| "finish_mpe_med": -28.22 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 0.04, |
| "turns_med": 6.0, |
| "err_total": 16, |
| "cost_total": 0.0, |
| "tokens_total": 296800, |
| "eac_ape_med": 1.08, |
| "eac_mpe_med": -1.08, |
| "finish_err_med": 0.0, |
| "finish_mpe_med": 0.0 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 0.0, |
| "turns_med": 6.0, |
| "err_total": 12, |
| "cost_total": 0.0, |
| "tokens_total": 147571 |
| } |
| } |
| }, |
| "student_MiniCPM5-1B_sft": { |
| "family": "agent", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 0.991, |
| "turns_med": 3.0, |
| "err_total": 66, |
| "cost_total": 0.0, |
| "tokens_total": 346514, |
| "eac_ape_med": 3.28, |
| "eac_mpe_med": -0.05, |
| "finish_err_med": 0.96, |
| "finish_mpe_med": 0.0 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 0.991, |
| "turns_med": 3.0, |
| "err_total": 93, |
| "cost_total": 0.0, |
| "tokens_total": 411124, |
| "eac_ape_med": 2.69, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 0.76, |
| "finish_mpe_med": 0.0 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 0.991, |
| "turns_med": 3.0, |
| "err_total": 98, |
| "cost_total": 0.0, |
| "tokens_total": 462523, |
| "eac_ape_med": 1.92, |
| "eac_mpe_med": -0.0, |
| "finish_err_med": 0.44, |
| "finish_mpe_med": 0.0 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 0.981, |
| "turns_med": 3.0, |
| "err_total": 136, |
| "cost_total": 0.0, |
| "tokens_total": 548791, |
| "eac_ape_med": 1.28, |
| "eac_mpe_med": 0.04, |
| "finish_err_med": 0.39, |
| "finish_mpe_med": 0.0 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 0.977, |
| "turns_med": 3.0, |
| "err_total": 37, |
| "cost_total": 0.0, |
| "tokens_total": 174178, |
| "eac_ape_med": 1.58, |
| "eac_mpe_med": 0.02, |
| "finish_err_med": 0.34, |
| "finish_mpe_med": 0.0 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 1.0, |
| "turns_med": 4.0, |
| "err_total": 28, |
| "cost_total": 0.0, |
| "tokens_total": 91391, |
| "eac_ape_med": 3.41, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 0.77, |
| "finish_mpe_med": 0.0 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 1.0, |
| "turns_med": 3.0, |
| "err_total": 20, |
| "cost_total": 0.0, |
| "tokens_total": 94454, |
| "eac_ape_med": 2.24, |
| "eac_mpe_med": -0.1, |
| "finish_err_med": 1.0, |
| "finish_mpe_med": -0.42 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "turns_med": 4.0, |
| "err_total": 8, |
| "cost_total": 0.0, |
| "tokens_total": 51101, |
| "eac_ape_med": 6.84, |
| "eac_mpe_med": 0.32, |
| "finish_err_med": 1.63, |
| "finish_mpe_med": -0.15 |
| } |
| } |
| }, |
| "student_Nemotron-4B_base": { |
| "family": "agent", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 0.991, |
| "turns_med": 10.0, |
| "err_total": 123, |
| "cost_total": 0.0, |
| "tokens_total": 235039, |
| "eac_ape_med": 3.32, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 1.14, |
| "finish_mpe_med": 0.0 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 0.972, |
| "turns_med": 9.0, |
| "err_total": 126, |
| "cost_total": 0.0, |
| "tokens_total": 240451, |
| "eac_ape_med": 2.69, |
| "eac_mpe_med": 0.02, |
| "finish_err_med": 1.0, |
| "finish_mpe_med": 0.05 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 0.953, |
| "turns_med": 10.0, |
| "err_total": 142, |
| "cost_total": 0.0, |
| "tokens_total": 368215, |
| "eac_ape_med": 1.84, |
| "eac_mpe_med": -0.0, |
| "finish_err_med": 0.51, |
| "finish_mpe_med": 0.0 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 0.972, |
| "turns_med": 10.0, |
| "err_total": 119, |
| "cost_total": 0.0, |
| "tokens_total": 268546, |
| "eac_ape_med": 1.47, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 0.47, |
| "finish_mpe_med": 0.15 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 1.0, |
| "turns_med": 9.0, |
| "err_total": 50, |
| "cost_total": 0.0, |
| "tokens_total": 106166, |
| "eac_ape_med": 1.34, |
| "eac_mpe_med": 0.69, |
| "finish_err_med": 1.0, |
| "finish_mpe_med": 5.49 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 0.963, |
| "turns_med": 9.0, |
| "err_total": 35, |
| "cost_total": 0.0, |
| "tokens_total": 54595, |
| "eac_ape_med": 3.79, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 0.94, |
| "finish_mpe_med": 0.0 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 0.92, |
| "turns_med": 10.0, |
| "err_total": 26, |
| "cost_total": 0.0, |
| "tokens_total": 50844, |
| "eac_ape_med": 2.48, |
| "eac_mpe_med": -0.1, |
| "finish_err_med": 0.61, |
| "finish_mpe_med": -0.42 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "turns_med": 12.0, |
| "err_total": 15, |
| "cost_total": 0.0, |
| "tokens_total": 28846, |
| "eac_ape_med": 6.32, |
| "eac_mpe_med": 0.74, |
| "finish_err_med": 1.63, |
| "finish_mpe_med": -2.04 |
| } |
| } |
| }, |
| "student_Qwen3.5-4B_base": { |
| "family": "agent", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 0.981, |
| "turns_med": 3.0, |
| "err_total": 200, |
| "cost_total": 0.0, |
| "tokens_total": 396281, |
| "eac_ape_med": 3.41, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 0.92, |
| "finish_mpe_med": 0.0 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "turns_med": 3.0, |
| "err_total": 134, |
| "cost_total": 0.0, |
| "tokens_total": 384125, |
| "eac_ape_med": 2.78, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 0.61, |
| "finish_mpe_med": 0.0 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 0.981, |
| "turns_med": 3.0, |
| "err_total": 155, |
| "cost_total": 0.0, |
| "tokens_total": 376693, |
| "eac_ape_med": 1.61, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 0.36, |
| "finish_mpe_med": 0.0 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "turns_med": 3.0, |
| "err_total": 133, |
| "cost_total": 0.0, |
| "tokens_total": 381914, |
| "eac_ape_med": 1.31, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 0.39, |
| "finish_mpe_med": 0.23 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 1.0, |
| "turns_med": 3.0, |
| "err_total": 39, |
| "cost_total": 0.0, |
| "tokens_total": 136558, |
| "eac_ape_med": 1.54, |
| "eac_mpe_med": -0.0, |
| "finish_err_med": 0.51, |
| "finish_mpe_med": 0.0 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 1.0, |
| "turns_med": 4.0, |
| "err_total": 46, |
| "cost_total": 0.0, |
| "tokens_total": 112634, |
| "eac_ape_med": 4.17, |
| "eac_mpe_med": 0.92, |
| "finish_err_med": 0.29, |
| "finish_mpe_med": 0.0 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 1.0, |
| "turns_med": 3.0, |
| "err_total": 33, |
| "cost_total": 0.0, |
| "tokens_total": 88064, |
| "eac_ape_med": 2.02, |
| "eac_mpe_med": -0.1, |
| "finish_err_med": 0.9, |
| "finish_mpe_med": 0.0 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "turns_med": 4.0, |
| "err_total": 16, |
| "cost_total": 0.0, |
| "tokens_total": 46869, |
| "eac_ape_med": 7.24, |
| "eac_mpe_med": 0.32, |
| "finish_err_med": 1.31, |
| "finish_mpe_med": -2.04 |
| } |
| } |
| }, |
| "student_gemma-4-E2B_base": { |
| "family": "agent", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 0.673, |
| "turns_med": 6.0, |
| "err_total": 991, |
| "cost_total": 0.0, |
| "tokens_total": 929091, |
| "eac_ape_med": 3.6, |
| "eac_mpe_med": 0.86, |
| "finish_err_med": 0.91, |
| "finish_mpe_med": 0.56 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 0.664, |
| "turns_med": 6.0, |
| "err_total": 986, |
| "cost_total": 0.0, |
| "tokens_total": 922957, |
| "eac_ape_med": 3.21, |
| "eac_mpe_med": -0.0, |
| "finish_err_med": 0.75, |
| "finish_mpe_med": 0.0 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 0.729, |
| "turns_med": 8.0, |
| "err_total": 919, |
| "cost_total": 0.0, |
| "tokens_total": 946829, |
| "eac_ape_med": 1.79, |
| "eac_mpe_med": -0.06, |
| "finish_err_med": 0.38, |
| "finish_mpe_med": 1.1 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 0.682, |
| "turns_med": 6.0, |
| "err_total": 949, |
| "cost_total": 0.0, |
| "tokens_total": 940851, |
| "eac_ape_med": 1.51, |
| "eac_mpe_med": 0.13, |
| "finish_err_med": 0.27, |
| "finish_mpe_med": 1.16 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 0.674, |
| "turns_med": 5.0, |
| "err_total": 376, |
| "cost_total": 0.0, |
| "tokens_total": 350372, |
| "eac_ape_med": 2.37, |
| "eac_mpe_med": 0.44, |
| "finish_err_med": 0.66, |
| "finish_mpe_med": 11.05 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 0.667, |
| "turns_med": 7.0, |
| "err_total": 270, |
| "cost_total": 0.0, |
| "tokens_total": 232948, |
| "eac_ape_med": 4.17, |
| "eac_mpe_med": 0.11, |
| "finish_err_med": 0.23, |
| "finish_mpe_med": 0.0 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 0.72, |
| "turns_med": 5.0, |
| "err_total": 203, |
| "cost_total": 0.0, |
| "tokens_total": 200387, |
| "eac_ape_med": 2.96, |
| "eac_mpe_med": -0.2, |
| "finish_err_med": 0.71, |
| "finish_mpe_med": -0.67 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 0.5, |
| "turns_med": 23.5, |
| "err_total": 137, |
| "cost_total": 0.0, |
| "tokens_total": 139250, |
| "eac_ape_med": 23.22, |
| "eac_mpe_med": -9.25, |
| "finish_err_med": 2.18, |
| "finish_mpe_med": -2.9 |
| } |
| } |
| }, |
| "student_Qwen3.5-2B_base": { |
| "family": "agent", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 0.243, |
| "turns_med": 24.0, |
| "err_total": 1743, |
| "cost_total": 0.0, |
| "tokens_total": 1164878, |
| "eac_ape_med": 3.3, |
| "eac_mpe_med": 0.24, |
| "finish_err_med": 0.74, |
| "finish_mpe_med": 0.0 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 0.159, |
| "turns_med": 24.0, |
| "err_total": 1798, |
| "cost_total": 0.0, |
| "tokens_total": 1214780, |
| "eac_ape_med": 1.2, |
| "eac_mpe_med": 0.57, |
| "finish_err_med": 0.9, |
| "finish_mpe_med": 0.0 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 0.187, |
| "turns_med": 24.0, |
| "err_total": 1703, |
| "cost_total": 0.0, |
| "tokens_total": 1049668, |
| "eac_ape_med": 2.14, |
| "eac_mpe_med": 1.42, |
| "finish_err_med": 0.47, |
| "finish_mpe_med": 2.28 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 0.196, |
| "turns_med": 24.0, |
| "err_total": 1698, |
| "cost_total": 0.0, |
| "tokens_total": 1539312, |
| "eac_ape_med": 2.42, |
| "eac_mpe_med": 0.11, |
| "finish_err_med": 0.09, |
| "finish_mpe_med": 0.32 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 0.209, |
| "turns_med": 24.0, |
| "err_total": 704, |
| "cost_total": 0.0, |
| "tokens_total": 499581, |
| "eac_ape_med": 1.2, |
| "eac_mpe_med": 0.58, |
| "finish_err_med": 0.9, |
| "finish_mpe_med": 0.0 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 0.185, |
| "turns_med": 24.0, |
| "err_total": 435, |
| "cost_total": 0.0, |
| "tokens_total": 422007, |
| "eac_ape_med": 4.17, |
| "eac_mpe_med": 0.92, |
| "finish_err_med": 0.77, |
| "finish_mpe_med": 3.86 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 0.12, |
| "turns_med": 24.0, |
| "err_total": 424, |
| "cost_total": 0.0, |
| "tokens_total": 203585, |
| "eac_ape_med": 0.57, |
| "eac_mpe_med": 0.34, |
| "finish_err_med": 3.39, |
| "finish_mpe_med": -19.96 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 0.0, |
| "turns_med": 24.0, |
| "err_total": 235, |
| "cost_total": 0.0, |
| "tokens_total": 89607 |
| } |
| } |
| }, |
| "student_Qwen3.5-4B_sft": { |
| "family": "agent", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 0.981, |
| "turns_med": 3.0, |
| "err_total": 143, |
| "cost_total": 0.0, |
| "tokens_total": 367727, |
| "eac_ape_med": 3.32, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 0.89, |
| "finish_mpe_med": 0.0 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "turns_med": 3.0, |
| "err_total": 129, |
| "cost_total": 0.0, |
| "tokens_total": 361918, |
| "eac_ape_med": 2.91, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 0.55, |
| "finish_mpe_med": 0.0 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 0.991, |
| "turns_med": 3.0, |
| "err_total": 122, |
| "cost_total": 0.0, |
| "tokens_total": 398920, |
| "eac_ape_med": 1.84, |
| "eac_mpe_med": -0.0, |
| "finish_err_med": 0.39, |
| "finish_mpe_med": 0.13 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "turns_med": 3.0, |
| "err_total": 131, |
| "cost_total": 0.0, |
| "tokens_total": 386452, |
| "eac_ape_med": 1.26, |
| "eac_mpe_med": -0.0, |
| "finish_err_med": 0.32, |
| "finish_mpe_med": 0.0 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 1.0, |
| "turns_med": 3.0, |
| "err_total": 49, |
| "cost_total": 0.0, |
| "tokens_total": 133298, |
| "eac_ape_med": 1.54, |
| "eac_mpe_med": -0.0, |
| "finish_err_med": 0.35, |
| "finish_mpe_med": 0.0 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 1.0, |
| "turns_med": 4.0, |
| "err_total": 48, |
| "cost_total": 0.0, |
| "tokens_total": 101768, |
| "eac_ape_med": 4.17, |
| "eac_mpe_med": 0.22, |
| "finish_err_med": 0.52, |
| "finish_mpe_med": 0.0 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 1.0, |
| "turns_med": 3.0, |
| "err_total": 26, |
| "cost_total": 0.0, |
| "tokens_total": 88527, |
| "eac_ape_med": 2.25, |
| "eac_mpe_med": -0.3, |
| "finish_err_med": 0.9, |
| "finish_mpe_med": -0.42 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "turns_med": 2.0, |
| "err_total": 6, |
| "cost_total": 0.0, |
| "tokens_total": 38325, |
| "eac_ape_med": 6.05, |
| "eac_mpe_med": 0.32, |
| "finish_err_med": 1.31, |
| "finish_mpe_med": -2.04 |
| } |
| } |
| }, |
| "student_gemma-4-E2B_sft": { |
| "family": "agent", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "turns_med": 3.0, |
| "err_total": 29, |
| "cost_total": 0.0, |
| "tokens_total": 494640, |
| "eac_ape_med": 3.43, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 0.79, |
| "finish_mpe_med": 0.0 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 0.991, |
| "turns_med": 3.0, |
| "err_total": 31, |
| "cost_total": 0.0, |
| "tokens_total": 541123, |
| "eac_ape_med": 2.31, |
| "eac_mpe_med": -0.0, |
| "finish_err_med": 0.63, |
| "finish_mpe_med": 0.0 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "turns_med": 3.0, |
| "err_total": 23, |
| "cost_total": 0.0, |
| "tokens_total": 548672, |
| "eac_ape_med": 1.76, |
| "eac_mpe_med": -0.0, |
| "finish_err_med": 0.38, |
| "finish_mpe_med": 0.0 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "turns_med": 3.0, |
| "err_total": 37, |
| "cost_total": 0.0, |
| "tokens_total": 558859, |
| "eac_ape_med": 1.28, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 0.39, |
| "finish_mpe_med": 0.0 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 1.0, |
| "turns_med": 3.0, |
| "err_total": 6, |
| "cost_total": 0.0, |
| "tokens_total": 202140, |
| "eac_ape_med": 1.54, |
| "eac_mpe_med": -0.0, |
| "finish_err_med": 0.35, |
| "finish_mpe_med": 0.0 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 1.0, |
| "turns_med": 3.0, |
| "err_total": 11, |
| "cost_total": 0.0, |
| "tokens_total": 126962, |
| "eac_ape_med": 3.41, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 0.52, |
| "finish_mpe_med": 0.0 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 0.96, |
| "turns_med": 3.0, |
| "err_total": 3, |
| "cost_total": 0.0, |
| "tokens_total": 128406, |
| "eac_ape_med": 2.07, |
| "eac_mpe_med": -0.2, |
| "finish_err_med": 1.05, |
| "finish_mpe_med": -0.21 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "turns_med": 3.0, |
| "err_total": 11, |
| "cost_total": 0.0, |
| "tokens_total": 83615, |
| "eac_ape_med": 6.84, |
| "eac_mpe_med": -1.43, |
| "finish_err_med": 1.63, |
| "finish_mpe_med": -2.04 |
| } |
| } |
| }, |
| "student_Qwen3.5-2B_sft": { |
| "family": "agent", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 0.271, |
| "turns_med": 24.0, |
| "err_total": 1653, |
| "cost_total": 0.0, |
| "tokens_total": 888476, |
| "eac_ape_med": 3.15, |
| "eac_mpe_med": 0.47, |
| "finish_err_med": 0.57, |
| "finish_mpe_med": 0.0 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 0.131, |
| "turns_med": 24.0, |
| "err_total": 1763, |
| "cost_total": 0.0, |
| "tokens_total": 1048427, |
| "eac_ape_med": 3.94, |
| "eac_mpe_med": 1.42, |
| "finish_err_med": 0.97, |
| "finish_mpe_med": 0.0 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 0.224, |
| "turns_med": 24.0, |
| "err_total": 1819, |
| "cost_total": 0.0, |
| "tokens_total": 1084626, |
| "eac_ape_med": 2.1, |
| "eac_mpe_med": 0.54, |
| "finish_err_med": 0.53, |
| "finish_mpe_med": -0.17 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 0.121, |
| "turns_med": 24.0, |
| "err_total": 1794, |
| "cost_total": 0.0, |
| "tokens_total": 1210331, |
| "eac_ape_med": 2.93, |
| "eac_mpe_med": 0.13, |
| "finish_err_med": 0.21, |
| "finish_mpe_med": 0.8 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 0.07, |
| "turns_med": 24.0, |
| "err_total": 787, |
| "cost_total": 0.0, |
| "tokens_total": 494830, |
| "eac_ape_med": 1.96, |
| "eac_mpe_med": 0.88, |
| "finish_err_med": 0.98, |
| "finish_mpe_med": 0.0 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 0.185, |
| "turns_med": 24.0, |
| "err_total": 418, |
| "cost_total": 0.0, |
| "tokens_total": 188645, |
| "eac_ape_med": 8.91, |
| "eac_mpe_med": 8.91, |
| "finish_err_med": 0.29, |
| "finish_mpe_med": 0.0 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 0.12, |
| "turns_med": 24.0, |
| "err_total": 383, |
| "cost_total": 0.0, |
| "tokens_total": 224868, |
| "eac_ape_med": 2.59, |
| "eac_mpe_med": 2.48, |
| "finish_err_med": 0.96, |
| "finish_mpe_med": 1.96 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 0.25, |
| "turns_med": 24.0, |
| "err_total": 175, |
| "cost_total": 0.0, |
| "tokens_total": 140084, |
| "eac_ape_med": 16.16, |
| "eac_mpe_med": 0.83, |
| "finish_err_med": 2.8, |
| "finish_mpe_med": -2.6 |
| } |
| } |
| }, |
| "student_Nemotron-4B_sft": { |
| "family": "agent", |
| "stages": { |
| "0.25": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "turns_med": 3.0, |
| "err_total": 35, |
| "cost_total": 0.0, |
| "tokens_total": 392676, |
| "eac_ape_med": 3.37, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 0.93, |
| "finish_mpe_med": 0.0 |
| }, |
| "0.40": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "turns_med": 3.0, |
| "err_total": 50, |
| "cost_total": 0.0, |
| "tokens_total": 395269, |
| "eac_ape_med": 2.37, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 0.61, |
| "finish_mpe_med": 0.0 |
| }, |
| "0.60": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "turns_med": 3.0, |
| "err_total": 30, |
| "cost_total": 0.0, |
| "tokens_total": 430285, |
| "eac_ape_med": 1.61, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 0.37, |
| "finish_mpe_med": 0.0 |
| }, |
| "0.75": { |
| "n": 107, |
| "valid_rate": 1.0, |
| "turns_med": 3.0, |
| "err_total": 41, |
| "cost_total": 0.0, |
| "tokens_total": 407593, |
| "eac_ape_med": 1.28, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 0.29, |
| "finish_mpe_med": 0.0 |
| } |
| }, |
| "by_strata": { |
| "short": { |
| "n": 43, |
| "valid_rate": 1.0, |
| "turns_med": 3.0, |
| "err_total": 10, |
| "cost_total": 0.0, |
| "tokens_total": 134419, |
| "eac_ape_med": 1.54, |
| "eac_mpe_med": -0.0, |
| "finish_err_med": 0.36, |
| "finish_mpe_med": 0.0 |
| }, |
| "medium": { |
| "n": 27, |
| "valid_rate": 1.0, |
| "turns_med": 3.0, |
| "err_total": 13, |
| "cost_total": 0.0, |
| "tokens_total": 101902, |
| "eac_ape_med": 3.41, |
| "eac_mpe_med": 0.0, |
| "finish_err_med": 0.77, |
| "finish_mpe_med": 0.0 |
| }, |
| "long": { |
| "n": 25, |
| "valid_rate": 1.0, |
| "turns_med": 3.0, |
| "err_total": 6, |
| "cost_total": 0.0, |
| "tokens_total": 97588, |
| "eac_ape_med": 2.02, |
| "eac_mpe_med": -0.1, |
| "finish_err_med": 0.88, |
| "finish_mpe_med": -0.43 |
| }, |
| "very_long": { |
| "n": 12, |
| "valid_rate": 1.0, |
| "turns_med": 4.0, |
| "err_total": 21, |
| "cost_total": 0.0, |
| "tokens_total": 61360, |
| "eac_ape_med": 7.23, |
| "eac_mpe_med": 0.45, |
| "finish_err_med": 1.63, |
| "finish_mpe_med": -2.04 |
| } |
| } |
| } |
| } |
| } |
