Add benchmark/: raw per-asset eval data (all models), reproduce.ipynb, per-timeframe charts
c5fc3bf verified | { | |
| "model": "AutoETS:7", | |
| "data": "data/subnet_ft2/holdout_1d.npz", | |
| "levels": [ | |
| 0.1, | |
| 0.2, | |
| 0.3, | |
| 0.4, | |
| 0.5, | |
| 0.6, | |
| 0.7, | |
| 0.8, | |
| 0.9 | |
| ], | |
| "context": 256, | |
| "horizons": { | |
| "30": { | |
| "n_assets": 109, | |
| "geomean_mase": 2.366563349925071, | |
| "geomean_ratio": 0.6919476942590365, | |
| "median_ratio": 0.8114673390491344, | |
| "beat_naive_frac": 0.8623853211009175, | |
| "per_asset": { | |
| "TAOUSD": { | |
| "ratio": 0.6675706337178313, | |
| "mase": 1.4918900332288945, | |
| "n_origins": 10 | |
| }, | |
| "sn1": { | |
| "ratio": 0.8461440507330507, | |
| "mase": 3.203374959172397, | |
| "n_origins": 10 | |
| }, | |
| "sn10": { | |
| "ratio": 0.8830150160022053, | |
| "mase": 4.199760379354215, | |
| "n_origins": 10 | |
| }, | |
| "sn100": { | |
| "ratio": 0.9431417918320143, | |
| "mase": 0.05503257864481561, | |
| "n_origins": 10 | |
| }, | |
| "sn101": { | |
| "ratio": 0.901056642068989, | |
| "mase": 0.060400293753296384, | |
| "n_origins": 10 | |
| }, | |
| "sn104": { | |
| "ratio": 0.8289208785402575, | |
| "mase": 9.672455965028094, | |
| "n_origins": 10 | |
| }, | |
| "sn105": { | |
| "ratio": 1.001057028708722, | |
| "mase": 2.670384030633187, | |
| "n_origins": 10 | |
| }, | |
| "sn106": { | |
| "ratio": 0.8249155301018027, | |
| "mase": 3.4018202835732816, | |
| "n_origins": 10 | |
| }, | |
| "sn107": { | |
| "ratio": 1.3353717207335187, | |
| "mase": 6.557065800315818, | |
| "n_origins": 10 | |
| }, | |
| "sn108": { | |
| "ratio": 0.9345372210722562, | |
| "mase": 1.9534784363854074, | |
| "n_origins": 10 | |
| }, | |
| "sn109": { | |
| "ratio": 0.7693761706733211, | |
| "mase": 1.9923048410699373, | |
| "n_origins": 10 | |
| }, | |
| "sn11": { | |
| "ratio": 0.7549674839683024, | |
| "mase": 2.7923225803969753, | |
| "n_origins": 10 | |
| }, | |
| "sn110": { | |
| "ratio": 0.9468760582553021, | |
| "mase": 0.6170583205651515, | |
| "n_origins": 10 | |
| }, | |
| "sn111": { | |
| "ratio": 0.8739339462711347, | |
| "mase": 13.533339158786571, | |
| "n_origins": 10 | |
| }, | |
| "sn112": { | |
| "ratio": 0.9733056526263412, | |
| "mase": 6.400781785118957, | |
| "n_origins": 10 | |
| }, | |
| "sn113": { | |
| "ratio": 1.0523455258237195, | |
| "mase": 2.492935023810832, | |
| "n_origins": 10 | |
| }, | |
| "sn114": { | |
| "ratio": 1.0255680942242271, | |
| "mase": 2.4080453645305573, | |
| "n_origins": 10 | |
| }, | |
| "sn115": { | |
| "ratio": 1.6684206393085632, | |
| "mase": 0.02499789448295681, | |
| "n_origins": 10 | |
| }, | |
| "sn117": { | |
| "ratio": 0.9239041211645671, | |
| "mase": 0.06753980220878117, | |
| "n_origins": 10 | |
| }, | |
| "sn118": { | |
| "ratio": 0.12855802882630832, | |
| "mase": 0.2689499709014488, | |
| "n_origins": 10 | |
| }, | |
| "sn119": { | |
| "ratio": 0.018084870334081963, | |
| "mase": 0.027401504650751946, | |
| "n_origins": 10 | |
| }, | |
| "sn12": { | |
| "ratio": 0.9603156443277242, | |
| "mase": 4.044310027197457, | |
| "n_origins": 10 | |
| }, | |
| "sn120": { | |
| "ratio": 0.5913936136551898, | |
| "mase": 1.768048536903948, | |
| "n_origins": 10 | |
| }, | |
| "sn121": { | |
| "ratio": 0.6047484387149469, | |
| "mase": 1.2658103519520179, | |
| "n_origins": 10 | |
| }, | |
| "sn123": { | |
| "ratio": 1.0978750453056907, | |
| "mase": 1.9007489247914706, | |
| "n_origins": 10 | |
| }, | |
| "sn124": { | |
| "ratio": 1.35319257178729, | |
| "mase": 5.134421915501078, | |
| "n_origins": 10 | |
| }, | |
| "sn125": { | |
| "ratio": 1.0595216426407497, | |
| "mase": 3.619624317060722, | |
| "n_origins": 10 | |
| }, | |
| "sn126": { | |
| "ratio": 0.8471152222379582, | |
| "mase": 3.8769852101389066, | |
| "n_origins": 10 | |
| }, | |
| "sn127": { | |
| "ratio": 0.827979286960206, | |
| "mase": 1.3375882065638716, | |
| "n_origins": 10 | |
| }, | |
| "sn128": { | |
| "ratio": 0.7839833124090737, | |
| "mase": 5.451304686329101, | |
| "n_origins": 10 | |
| }, | |
| "sn13": { | |
| "ratio": 0.7774680387632648, | |
| "mase": 2.2069030401913046, | |
| "n_origins": 10 | |
| }, | |
| "sn14": { | |
| "ratio": 0.9655946589257736, | |
| "mase": 8.87690847045376, | |
| "n_origins": 10 | |
| }, | |
| "sn15": { | |
| "ratio": 0.6628273466450806, | |
| "mase": 2.963345623806144, | |
| "n_origins": 10 | |
| }, | |
| "sn17": { | |
| "ratio": 0.5837911999609318, | |
| "mase": 3.3639883924145635, | |
| "n_origins": 10 | |
| }, | |
| "sn18": { | |
| "ratio": 0.9660904134323536, | |
| "mase": 3.790243830263383, | |
| "n_origins": 10 | |
| }, | |
| "sn19": { | |
| "ratio": 0.7294916494486682, | |
| "mase": 3.1126939066627815, | |
| "n_origins": 10 | |
| }, | |
| "sn2": { | |
| "ratio": 0.6651274178841454, | |
| "mase": 1.2102657796220329, | |
| "n_origins": 10 | |
| }, | |
| "sn20": { | |
| "ratio": 0.7442371359351135, | |
| "mase": 3.378529960749457, | |
| "n_origins": 10 | |
| }, | |
| "sn21": { | |
| "ratio": 0.8198638697518402, | |
| "mase": 4.369837044642523, | |
| "n_origins": 10 | |
| }, | |
| "sn22": { | |
| "ratio": 0.7588113749293093, | |
| "mase": 2.385519611103995, | |
| "n_origins": 10 | |
| }, | |
| "sn23": { | |
| "ratio": 1.0439720539117399, | |
| "mase": 7.097410345623546, | |
| "n_origins": 10 | |
| }, | |
| "sn24": { | |
| "ratio": 0.8893609290612936, | |
| "mase": 3.4680109625321314, | |
| "n_origins": 10 | |
| }, | |
| "sn25": { | |
| "ratio": 0.9428539211957584, | |
| "mase": 17.80637976196257, | |
| "n_origins": 10 | |
| }, | |
| "sn27": { | |
| "ratio": 0.8992728968434015, | |
| "mase": 4.8782912089303085, | |
| "n_origins": 10 | |
| }, | |
| "sn28": { | |
| "ratio": 0.8924076741454869, | |
| "mase": 6.866075941640846, | |
| "n_origins": 10 | |
| }, | |
| "sn29": { | |
| "ratio": 0.8219901180416586, | |
| "mase": 1.181282867403581, | |
| "n_origins": 10 | |
| }, | |
| "sn3": { | |
| "ratio": 0.4762046073655373, | |
| "mase": 1.247610231615908, | |
| "n_origins": 10 | |
| }, | |
| "sn30": { | |
| "ratio": 0.7879375904750887, | |
| "mase": 3.5319859212399307, | |
| "n_origins": 10 | |
| }, | |
| "sn31": { | |
| "ratio": 0.9119319924469904, | |
| "mase": 2.6862966713979133, | |
| "n_origins": 10 | |
| }, | |
| "sn32": { | |
| "ratio": 0.7909164122701355, | |
| "mase": 1.4493120454358026, | |
| "n_origins": 10 | |
| }, | |
| "sn33": { | |
| "ratio": 0.6395058601473856, | |
| "mase": 1.8350381614483284, | |
| "n_origins": 10 | |
| }, | |
| "sn34": { | |
| "ratio": 0.58128081462755, | |
| "mase": 3.674982404327822, | |
| "n_origins": 10 | |
| }, | |
| "sn35": { | |
| "ratio": 0.6752239463417619, | |
| "mase": 1.5508054468600845, | |
| "n_origins": 10 | |
| }, | |
| "sn37": { | |
| "ratio": 0.9306814560931853, | |
| "mase": 3.3213476364552257, | |
| "n_origins": 10 | |
| }, | |
| "sn38": { | |
| "ratio": 0.5899380657679953, | |
| "mase": 4.60659119666794, | |
| "n_origins": 10 | |
| }, | |
| "sn39": { | |
| "ratio": 0.30620842930559034, | |
| "mase": 0.7578587649038698, | |
| "n_origins": 10 | |
| }, | |
| "sn4": { | |
| "ratio": 0.8947379652535096, | |
| "mase": 3.988734300737364, | |
| "n_origins": 10 | |
| }, | |
| "sn41": { | |
| "ratio": 0.5624213285505781, | |
| "mase": 1.5610196633588571, | |
| "n_origins": 10 | |
| }, | |
| "sn42": { | |
| "ratio": 0.772447438996406, | |
| "mase": 2.891367491399924, | |
| "n_origins": 10 | |
| }, | |
| "sn43": { | |
| "ratio": 0.7310382344768022, | |
| "mase": 2.465246165169788, | |
| "n_origins": 10 | |
| }, | |
| "sn44": { | |
| "ratio": 0.694900273575232, | |
| "mase": 3.600180971546157, | |
| "n_origins": 10 | |
| }, | |
| "sn45": { | |
| "ratio": 0.9733196343261465, | |
| "mase": 1.4488641186809776, | |
| "n_origins": 10 | |
| }, | |
| "sn46": { | |
| "ratio": 0.9491776045055867, | |
| "mase": 6.4428370872316565, | |
| "n_origins": 10 | |
| }, | |
| "sn47": { | |
| "ratio": 0.6998357809839778, | |
| "mase": 2.078531638018453, | |
| "n_origins": 10 | |
| }, | |
| "sn48": { | |
| "ratio": 0.44835346987560354, | |
| "mase": 1.9396244528947353, | |
| "n_origins": 10 | |
| }, | |
| "sn49": { | |
| "ratio": 0.9583421899664988, | |
| "mase": 3.3627057114489096, | |
| "n_origins": 10 | |
| }, | |
| "sn5": { | |
| "ratio": 0.9312521420037191, | |
| "mase": 4.2470904627526505, | |
| "n_origins": 10 | |
| }, | |
| "sn50": { | |
| "ratio": 0.7803806098144638, | |
| "mase": 1.6201332967017958, | |
| "n_origins": 10 | |
| }, | |
| "sn51": { | |
| "ratio": 0.8607138572893575, | |
| "mase": 11.490897907935876, | |
| "n_origins": 10 | |
| }, | |
| "sn52": { | |
| "ratio": 0.671600934708315, | |
| "mase": 2.7521099455749853, | |
| "n_origins": 10 | |
| }, | |
| "sn53": { | |
| "ratio": 0.8036462185422981, | |
| "mase": 101.89765302109727, | |
| "n_origins": 10 | |
| }, | |
| "sn54": { | |
| "ratio": 0.7377592875792981, | |
| "mase": 2.373034686929311, | |
| "n_origins": 10 | |
| }, | |
| "sn55": { | |
| "ratio": 0.6596951110881158, | |
| "mase": 5.831668088551132, | |
| "n_origins": 10 | |
| }, | |
| "sn56": { | |
| "ratio": 0.6975730100405195, | |
| "mase": 3.5871928590426476, | |
| "n_origins": 10 | |
| }, | |
| "sn6": { | |
| "ratio": 0.4474750424621897, | |
| "mase": 1.3088350451348267, | |
| "n_origins": 10 | |
| }, | |
| "sn60": { | |
| "ratio": 0.9688464568986638, | |
| "mase": 4.957461564781218, | |
| "n_origins": 10 | |
| }, | |
| "sn61": { | |
| "ratio": 0.9851475595329916, | |
| "mase": 7.638887482563111, | |
| "n_origins": 10 | |
| }, | |
| "sn62": { | |
| "ratio": 0.18957647119470564, | |
| "mase": 0.6935409849581684, | |
| "n_origins": 10 | |
| }, | |
| "sn63": { | |
| "ratio": 0.8109767593881151, | |
| "mase": 1.643743917758603, | |
| "n_origins": 10 | |
| }, | |
| "sn64": { | |
| "ratio": 0.7998135302551915, | |
| "mase": 5.777014710107664, | |
| "n_origins": 10 | |
| }, | |
| "sn65": { | |
| "ratio": 0.7575287932017414, | |
| "mase": 3.315467124961679, | |
| "n_origins": 10 | |
| }, | |
| "sn66": { | |
| "ratio": 0.9630316907764966, | |
| "mase": 5.433615778172927, | |
| "n_origins": 10 | |
| }, | |
| "sn67": { | |
| "ratio": 0.008067211404894064, | |
| "mase": 0.04617162476333115, | |
| "n_origins": 10 | |
| }, | |
| "sn68": { | |
| "ratio": 0.7250884463868913, | |
| "mase": 2.8780304246885753, | |
| "n_origins": 10 | |
| }, | |
| "sn7": { | |
| "ratio": 0.8228739162896898, | |
| "mase": 2.317613816468815, | |
| "n_origins": 10 | |
| }, | |
| "sn71": { | |
| "ratio": 1.0239235890456015, | |
| "mase": 2.161939992585475, | |
| "n_origins": 10 | |
| }, | |
| "sn72": { | |
| "ratio": 0.6311912557372256, | |
| "mase": 2.6603161071538284, | |
| "n_origins": 10 | |
| }, | |
| "sn73": { | |
| "ratio": 0.7908497042718189, | |
| "mase": 1.5486374986856677, | |
| "n_origins": 10 | |
| }, | |
| "sn74": { | |
| "ratio": 0.8114673390491344, | |
| "mase": 3.700347834984087, | |
| "n_origins": 10 | |
| }, | |
| "sn75": { | |
| "ratio": 0.5139534927209694, | |
| "mase": 2.0490422453734154, | |
| "n_origins": 10 | |
| }, | |
| "sn76": { | |
| "ratio": 0.7202280264960904, | |
| "mase": 2.139687330629988, | |
| "n_origins": 10 | |
| }, | |
| "sn77": { | |
| "ratio": 1.5393112748731639, | |
| "mase": 22.02669709474096, | |
| "n_origins": 10 | |
| }, | |
| "sn79": { | |
| "ratio": 0.8070686761154677, | |
| "mase": 9.277222174208783, | |
| "n_origins": 10 | |
| }, | |
| "sn8": { | |
| "ratio": 0.4808211391055732, | |
| "mase": 2.2057639834055456, | |
| "n_origins": 10 | |
| }, | |
| "sn80": { | |
| "ratio": 0.9667850037495781, | |
| "mase": 13.04083355215854, | |
| "n_origins": 10 | |
| }, | |
| "sn81": { | |
| "ratio": 0.39566553963790496, | |
| "mase": 1.3533832646878636, | |
| "n_origins": 10 | |
| }, | |
| "sn83": { | |
| "ratio": 1.0014947772469125, | |
| "mase": 13.092697142363836, | |
| "n_origins": 10 | |
| }, | |
| "sn85": { | |
| "ratio": 0.7158424302109977, | |
| "mase": 1.6191896045216625, | |
| "n_origins": 10 | |
| }, | |
| "sn87": { | |
| "ratio": 0.9719152958969617, | |
| "mase": 4.593749803177495, | |
| "n_origins": 10 | |
| }, | |
| "sn88": { | |
| "ratio": 0.7799638936554872, | |
| "mase": 1.887729847068682, | |
| "n_origins": 10 | |
| }, | |
| "sn89": { | |
| "ratio": 1.00672702947416, | |
| "mase": 12.825304740593516, | |
| "n_origins": 10 | |
| }, | |
| "sn9": { | |
| "ratio": 0.9451005241969795, | |
| "mase": 5.915718921074402, | |
| "n_origins": 10 | |
| }, | |
| "sn91": { | |
| "ratio": 0.790079594957739, | |
| "mase": 3.81294369482132, | |
| "n_origins": 10 | |
| }, | |
| "sn93": { | |
| "ratio": 0.8373692246781106, | |
| "mase": 2.4092574666777717, | |
| "n_origins": 10 | |
| }, | |
| "sn94": { | |
| "ratio": 0.8631383490492676, | |
| "mase": 0.13679950608799363, | |
| "n_origins": 10 | |
| }, | |
| "sn95": { | |
| "ratio": 1.1402112560238669, | |
| "mase": 6.4857085703263015, | |
| "n_origins": 10 | |
| }, | |
| "sn96": { | |
| "ratio": 0.00463236554065909, | |
| "mase": 0.07301759429463427, | |
| "n_origins": 10 | |
| }, | |
| "sn97": { | |
| "ratio": 0.65055568584658, | |
| "mase": 1.1776962901878116, | |
| "n_origins": 10 | |
| }, | |
| "sn98": { | |
| "ratio": 1.1064400444391076, | |
| "mase": 2.4840879007160175, | |
| "n_origins": 10 | |
| } | |
| } | |
| } | |
| } | |
| } |