| { |
| "model": "Qwen/Qwen3-4B-Instruct-2507", |
| "ckpt": "/tmp/checkpoints/search_step_2000.pt", |
| "by_K": { |
| "16": { |
| "raw_qk": { |
| "per_layer": { |
| "4": 0.4115331669648488, |
| "8": 0.2802686393260956, |
| "12": 0.1508728675544262, |
| "16": 0.07771651136378448, |
| "20": 0.053202067812283836, |
| "24": 0.10271163408954938 |
| }, |
| "avg": 0.1793841478518314 |
| }, |
| "learned": { |
| "per_layer": { |
| "4": 0.1072007929906249, |
| "8": 0.23697869976361594, |
| "12": 0.11224483884871006, |
| "16": 0.07637482260664304, |
| "20": 0.06903641019016504, |
| "24": 0.10397349204868078 |
| }, |
| "avg": 0.11763484274140662 |
| } |
| }, |
| "32": { |
| "raw_qk": { |
| "per_layer": { |
| "4": 0.4668281575043996, |
| "8": 0.3293568876882394, |
| "12": 0.19662011042237282, |
| "16": 0.11227216385304928, |
| "20": 0.07988839099804561, |
| "24": 0.144585732370615 |
| }, |
| "avg": 0.22159190713945362 |
| }, |
| "learned": { |
| "per_layer": { |
| "4": 0.13327929687996706, |
| "8": 0.2569987513124943, |
| "12": 0.1446449818710486, |
| "16": 0.10432848272224267, |
| "20": 0.09580977975080411, |
| "24": 0.13778831561406454 |
| }, |
| "avg": 0.14547493469177022 |
| } |
| }, |
| "64": { |
| "raw_qk": { |
| "per_layer": { |
| "4": 0.5168702056010565, |
| "8": 0.390024371445179, |
| "12": 0.25787363573908806, |
| "16": 0.15796820322672525, |
| "20": 0.11998403631150723, |
| "24": 0.2020296814541022 |
| }, |
| "avg": 0.2741250222962764 |
| }, |
| "learned": { |
| "per_layer": { |
| "4": 0.1732035626967748, |
| "8": 0.2893482334911823, |
| "12": 0.19321986908713976, |
| "16": 0.14695298795898756, |
| "20": 0.13634028658270836, |
| "24": 0.1862101349979639 |
| }, |
| "avg": 0.18754584580245945 |
| } |
| }, |
| "128": { |
| "raw_qk": { |
| "per_layer": { |
| "4": 0.571592112382253, |
| "8": 0.463805615901947, |
| "12": 0.33948806673288345, |
| "16": 0.22183777391910553, |
| "20": 0.18010229741533598, |
| "24": 0.27901028965910274 |
| }, |
| "avg": 0.34263935933510464 |
| }, |
| "learned": { |
| "per_layer": { |
| "4": 0.235384251922369, |
| "8": 0.3402557211617629, |
| "12": 0.2643987759947777, |
| "16": 0.21102494125564894, |
| "20": 0.19678996006647745, |
| "24": 0.25442706421017647 |
| }, |
| "avg": 0.25038011910186875 |
| } |
| }, |
| "256": { |
| "raw_qk": { |
| "per_layer": { |
| "4": 0.6327291478713354, |
| "8": 0.5521376008788744, |
| "12": 0.43828647087017697, |
| "16": 0.31400871525208157, |
| "20": 0.2687250425418218, |
| "24": 0.37980743249257404 |
| }, |
| "avg": 0.4309490683178107 |
| }, |
| "learned": { |
| "per_layer": { |
| "4": 0.3288092017173767, |
| "8": 0.41752680391073227, |
| "12": 0.36623430997133255, |
| "16": 0.30524607251087826, |
| "20": 0.2853706416984399, |
| "24": 0.34850213179985684 |
| }, |
| "avg": 0.3419481936014361 |
| } |
| } |
| }, |
| "learned_over_raw_K128": 0.7307395145371917 |
| } |
