Hugging Face's logo

datasysdev
/
ann-sparseattention

Transformers
sparse-attention
approximate-nearest-neighbors
faiss
qwen3
long-context
Model card Files
xet
 Community
datasysdev commited on
Commit
56136c5
·
verified ·
1 Parent(s): cb2df3b

Upload checkpoints_all36_d128_block/search_step_750.compare_retrieval.json

Browse files
Files changed (1) hide show
  1. checkpoints_all36_d128_block/search_step_750.compare_retrieval.json +175 -0
checkpoints_all36_d128_block/search_step_750.compare_retrieval.json ADDED
@@ -0,0 +1,175 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": "Qwen/Qwen3-4B-Instruct-2507",
3
+ "ckpt": "/tmp/checkpoints_all36_d128_block/search_step_750.pt",
4
+ "by_K": {
5
+ "128": {
6
+ "raw_qk": {
7
+ "per_layer": {
8
+ "0": 0.9221310317516327,
9
+ "1": 0.9183070361614227,
10
+ "2": 0.9386252164840698,
11
+ "3": 0.9387637376785278,
12
+ "4": 0.9440614581108093,
13
+ "5": 0.96429443359375,
14
+ "6": 0.9560344219207764,
15
+ "7": 0.9818542897701263,
16
+ "8": 0.9707900285720825,
17
+ "9": 0.9587394297122955,
18
+ "10": 0.9742867350578308,
19
+ "11": 0.9763059914112091,
20
+ "12": 0.9611693322658539,
21
+ "13": 0.9707654714584351,
22
+ "14": 0.9733296930789948,
23
+ "15": 0.9675203263759613,
24
+ "16": 0.9556995928287506,
25
+ "17": 0.9593449831008911,
26
+ "18": 0.9652450382709503,
27
+ "19": 0.9612957835197449,
28
+ "20": 0.9591256380081177,
29
+ "21": 0.9657162129878998,
30
+ "22": 0.962736189365387,
31
+ "23": 0.9788879752159119,
32
+ "24": 0.9710511267185211,
33
+ "25": 0.9860879778862,
34
+ "26": 0.9775076508522034,
35
+ "27": 0.977691113948822,
36
+ "28": 0.9794201850891113,
37
+ "29": 0.9821525514125824,
38
+ "30": 0.9875719547271729,
39
+ "31": 0.9842455685138702,
40
+ "32": 0.9791028797626495,
41
+ "33": 0.9772593379020691,
42
+ "34": 0.9759000241756439,
43
+ "35": 0.9796911776065826
44
+ },
45
+ "avg": 0.9661864332026906
46
+ },
47
+ "learned": {
48
+ "per_layer": {
49
+ "0": 0.7756117284297943,
50
+ "1": 0.8532524108886719,
51
+ "2": 0.898503303527832,
52
+ "3": 0.9247933924198151,
53
+ "4": 0.9357087314128876,
54
+ "5": 0.9496567249298096,
55
+ "6": 0.9385866224765778,
56
+ "7": 0.9828359186649323,
57
+ "8": 0.9707390367984772,
58
+ "9": 0.9764299988746643,
59
+ "10": 0.9706397652626038,
60
+ "11": 0.9758526980876923,
61
+ "12": 0.9699114561080933,
62
+ "13": 0.9716702997684479,
63
+ "14": 0.9731307923793793,
64
+ "15": 0.9723065197467804,
65
+ "16": 0.9619724154472351,
66
+ "17": 0.9667060077190399,
67
+ "18": 0.9725159108638763,
68
+ "19": 0.9684217274188995,
69
+ "20": 0.9755173027515411,
70
+ "21": 0.9796667695045471,
71
+ "22": 0.9698297381401062,
72
+ "23": 0.9845495820045471,
73
+ "24": 0.9784150421619415,
74
+ "25": 0.9885841608047485,
75
+ "26": 0.98563352227211,
76
+ "27": 0.9832673370838165,
77
+ "28": 0.9848426878452301,
78
+ "29": 0.987438291311264,
79
+ "30": 0.9862436354160309,
80
+ "31": 0.9839730858802795,
81
+ "32": 0.979794979095459,
82
+ "33": 0.9695486426353455,
83
+ "34": 0.960169792175293,
84
+ "35": 0.9648604094982147
85
+ },
86
+ "avg": 0.9603216788834996
87
+ }
88
+ },
89
+ "256": {
90
+ "raw_qk": {
91
+ "per_layer": {
92
+ "0": 0.9736533463001251,
93
+ "1": 0.9831015169620514,
94
+ "2": 0.985590249300003,
95
+ "3": 0.9856403172016144,
96
+ "4": 0.9863723814487457,
97
+ "5": 0.9927175343036652,
98
+ "6": 0.9899796843528748,
99
+ "7": 0.9961064755916595,
100
+ "8": 0.9944310188293457,
101
+ "9": 0.9924624264240265,
102
+ "10": 0.9945598244667053,
103
+ "11": 0.9956617951393127,
104
+ "12": 0.9914700388908386,
105
+ "13": 0.9947893917560577,
106
+ "14": 0.9956192076206207,
107
+ "15": 0.9947386384010315,
108
+ "16": 0.9918601512908936,
109
+ "17": 0.9932802319526672,
110
+ "18": 0.994801938533783,
111
+ "19": 0.9929636716842651,
112
+ "20": 0.9928486049175262,
113
+ "21": 0.9940518736839294,
114
+ "22": 0.9925838708877563,
115
+ "23": 0.9956205189228058,
116
+ "24": 0.9944405257701874,
117
+ "25": 0.99702388048172,
118
+ "26": 0.9963724911212921,
119
+ "27": 0.9952751994132996,
120
+ "28": 0.9952269494533539,
121
+ "29": 0.9970046579837799,
122
+ "30": 0.9976439476013184,
123
+ "31": 0.9967932999134064,
124
+ "32": 0.9949435591697693,
125
+ "33": 0.9954463541507721,
126
+ "34": 0.9949491024017334,
127
+ "35": 0.9950292408466339
128
+ },
129
+ "avg": 0.9929181643658214
130
+ },
131
+ "learned": {
132
+ "per_layer": {
133
+ "0": 0.9240421950817108,
134
+ "1": 0.9610750675201416,
135
+ "2": 0.9658757448196411,
136
+ "3": 0.9772964715957642,
137
+ "4": 0.9824097156524658,
138
+ "5": 0.9867623746395111,
139
+ "6": 0.9807166755199432,
140
+ "7": 0.9962040483951569,
141
+ "8": 0.9934569895267487,
142
+ "9": 0.9951488673686981,
143
+ "10": 0.9924411773681641,
144
+ "11": 0.9950867295265198,
145
+ "12": 0.9929095506668091,
146
+ "13": 0.9934256374835968,
147
+ "14": 0.9937738478183746,
148
+ "15": 0.9944608509540558,
149
+ "16": 0.9920228123664856,
150
+ "17": 0.9936496913433075,
151
+ "18": 0.9954988360404968,
152
+ "19": 0.9934418499469757,
153
+ "20": 0.9959242641925812,
154
+ "21": 0.9965861737728119,
155
+ "22": 0.9939501285552979,
156
+ "23": 0.9973835349082947,
157
+ "24": 0.9962238371372223,
158
+ "25": 0.9976089596748352,
159
+ "26": 0.9972057044506073,
160
+ "27": 0.9966239333152771,
161
+ "28": 0.9968054592609406,
162
+ "29": 0.997564047574997,
163
+ "30": 0.9967389404773712,
164
+ "31": 0.9965249300003052,
165
+ "32": 0.9954393804073334,
166
+ "33": 0.9919858574867249,
167
+ "34": 0.989928811788559,
168
+ "35": 0.9892888367176056
169
+ },
170
+ "avg": 0.9895967203709815
171
+ }
172
+ }
173
+ },
174
+ "learned_over_raw_K128": 0.9939299972369198
175
+ }