Hugging Face's logo

datasysdev
/
ann-sparseattention

Transformers
GGUF
sparse-attention
approximate-nearest-neighbors
faiss
qwen3
long-context
conversational
Model card Files
xet
 Community
ann-sparseattention
File size: 3,371 Bytes
57c6b5b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
{
  "model": "Qwen/Qwen3-4B-Instruct-2507",
  "ckpt": "/tmp/checkpoints/search_step_2000.pt",
  "by_K": {
    "16": {
      "raw_qk": {
        "per_layer": {
          "4": 0.4115331669648488,
          "8": 0.2802686393260956,
          "12": 0.1508728675544262,
          "16": 0.07771651136378448,
          "20": 0.053202067812283836,
          "24": 0.10271163408954938
        },
        "avg": 0.1793841478518314
      },
      "learned": {
        "per_layer": {
          "4": 0.1072007929906249,
          "8": 0.23697869976361594,
          "12": 0.11224483884871006,
          "16": 0.07637482260664304,
          "20": 0.06903641019016504,
          "24": 0.10397349204868078
        },
        "avg": 0.11763484274140662
      }
    },
    "32": {
      "raw_qk": {
        "per_layer": {
          "4": 0.4668281575043996,
          "8": 0.3293568876882394,
          "12": 0.19662011042237282,
          "16": 0.11227216385304928,
          "20": 0.07988839099804561,
          "24": 0.144585732370615
        },
        "avg": 0.22159190713945362
      },
      "learned": {
        "per_layer": {
          "4": 0.13327929687996706,
          "8": 0.2569987513124943,
          "12": 0.1446449818710486,
          "16": 0.10432848272224267,
          "20": 0.09580977975080411,
          "24": 0.13778831561406454
        },
        "avg": 0.14547493469177022
      }
    },
    "64": {
      "raw_qk": {
        "per_layer": {
          "4": 0.5168702056010565,
          "8": 0.390024371445179,
          "12": 0.25787363573908806,
          "16": 0.15796820322672525,
          "20": 0.11998403631150723,
          "24": 0.2020296814541022
        },
        "avg": 0.2741250222962764
      },
      "learned": {
        "per_layer": {
          "4": 0.1732035626967748,
          "8": 0.2893482334911823,
          "12": 0.19321986908713976,
          "16": 0.14695298795898756,
          "20": 0.13634028658270836,
          "24": 0.1862101349979639
        },
        "avg": 0.18754584580245945
      }
    },
    "128": {
      "raw_qk": {
        "per_layer": {
          "4": 0.571592112382253,
          "8": 0.463805615901947,
          "12": 0.33948806673288345,
          "16": 0.22183777391910553,
          "20": 0.18010229741533598,
          "24": 0.27901028965910274
        },
        "avg": 0.34263935933510464
      },
      "learned": {
        "per_layer": {
          "4": 0.235384251922369,
          "8": 0.3402557211617629,
          "12": 0.2643987759947777,
          "16": 0.21102494125564894,
          "20": 0.19678996006647745,
          "24": 0.25442706421017647
        },
        "avg": 0.25038011910186875
      }
    },
    "256": {
      "raw_qk": {
        "per_layer": {
          "4": 0.6327291478713354,
          "8": 0.5521376008788744,
          "12": 0.43828647087017697,
          "16": 0.31400871525208157,
          "20": 0.2687250425418218,
          "24": 0.37980743249257404
        },
        "avg": 0.4309490683178107
      },
      "learned": {
        "per_layer": {
          "4": 0.3288092017173767,
          "8": 0.41752680391073227,
          "12": 0.36623430997133255,
          "16": 0.30524607251087826,
          "20": 0.2853706416984399,
          "24": 0.34850213179985684
        },
        "avg": 0.3419481936014361
      }
    }
  },
  "learned_over_raw_K128": 0.7307395145371917
}