File size: 19,607 Bytes
38770a7 8c41d43 38770a7 8c41d43 adee57e 38770a7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 | {
"cases": [
{
"name": "bnb4-nf4-decode-m1-k2048-n2048-b64",
"preset": "smoke",
"vars": { "M": 1, "K": 2048, "N": 2048, "blockSize": 64 },
"attrs": { "K": 2048, "N": 2048, "block_size": 64, "quant_type": 1 },
"inputs": {
"aT": { "shape": [1, 2048], "dtype": "float32", "dist": "normal", "seed": 410, "scale": 0.2 },
"bT": { "shape": [2097152], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 },
"absmaxT": {
"shape": [65536],
"dtype": "float32",
"dist": "uniform",
"seed": 411,
"offset": 0.04,
"scale": 0.01,
"signed": false
}
},
"outputs": { "yT": { "shape": [1, 2048], "dtype": "float32" } },
"bench": {
"metrics": [
{
"type": "bandwidth",
"value": "numel(shapes.bT) * 4 + (numel(shapes.aT) + numel(shapes.absmaxT) + numel(shapes.yT)) * 4"
}
]
}
},
{
"name": "bnb4-nf4-prefill-m128-k2048-n2048-b64",
"preset": "smoke",
"vars": { "M": 128, "K": 2048, "N": 2048, "blockSize": 64 },
"attrs": { "K": 2048, "N": 2048, "block_size": 64, "quant_type": 1 },
"tunableSpace": { "PORTABLE_TILE_K": [16, 32, 64] },
"inputs": {
"aT": { "shape": [128, 2048], "dtype": "float32", "dist": "normal", "seed": 412, "scale": 0.2 },
"bT": { "shape": [2097152], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 },
"absmaxT": {
"shape": [65536],
"dtype": "float32",
"dist": "uniform",
"seed": 413,
"offset": 0.04,
"scale": 0.01,
"signed": false
}
},
"outputs": { "yT": { "shape": [128, 2048], "dtype": "float32" } },
"bench": {
"primary": true,
"metrics": [{ "type": "gflops", "value": "2 * dim(shapes.aT, 0) * args.K * args.N" }]
}
},
{
"name": "bnb4-fp4-prefill-m128-k2048-n2048-b64",
"preset": "smoke",
"vars": { "M": 128, "K": 2048, "N": 2048, "blockSize": 64 },
"attrs": { "K": 2048, "N": 2048, "block_size": 64, "quant_type": 0 },
"tunableSpace": { "PORTABLE_TILE_K": [16, 32, 64] },
"inputs": {
"aT": { "shape": [128, 2048], "dtype": "float32", "dist": "normal", "seed": 414, "scale": 0.2 },
"bT": { "shape": [2097152], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 },
"absmaxT": {
"shape": [65536],
"dtype": "float32",
"dist": "uniform",
"seed": 415,
"offset": 0.04,
"scale": 0.01,
"signed": false
}
},
"outputs": { "yT": { "shape": [128, 2048], "dtype": "float32" } },
"bench": { "metrics": [{ "type": "gflops", "value": "2 * dim(shapes.aT, 0) * args.K * args.N" }] }
},
{
"name": "bnb4-nf4-prefill-m512-k4096-n4096-b64",
"preset": "model",
"vars": { "M": 512, "K": 4096, "N": 4096, "blockSize": 64 },
"attrs": { "K": 4096, "N": 4096, "block_size": 64, "quant_type": 1 },
"inputs": {
"aT": { "shape": [512, 4096], "dtype": "float32", "dist": "normal", "seed": 416, "scale": 0.2 },
"bT": { "shape": [8388608], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 },
"absmaxT": {
"shape": [262144],
"dtype": "float32",
"dist": "uniform",
"seed": 417,
"offset": 0.04,
"scale": 0.01,
"signed": false
}
},
"outputs": { "yT": { "shape": [512, 4096], "dtype": "float32" } },
"bench": { "metrics": [{ "type": "gflops", "value": "2 * dim(shapes.aT, 0) * args.K * args.N" }] },
"tunableSpace": { "PORTABLE_TILE_K": [16, 32, 64] }
},
{
"name": "bnb4-nf4-prefill-m512-k4096-n4096-b64-f16",
"preset": "model",
"vars": { "M": 512, "K": 4096, "N": 4096, "blockSize": 64 },
"attrs": { "K": 4096, "N": 4096, "block_size": 64, "quant_type": 1 },
"inputs": {
"aT": { "shape": [512, 4096], "dtype": "float16", "dist": "normal", "seed": 416, "scale": 0.2 },
"bT": { "shape": [8388608], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 },
"absmaxT": {
"shape": [262144],
"dtype": "float16",
"dist": "uniform",
"seed": 417,
"offset": 0.04,
"scale": 0.01,
"signed": false
}
},
"outputs": { "yT": { "shape": [512, 4096], "dtype": "float16" } },
"bench": { "metrics": [{ "type": "gflops", "value": "2 * dim(shapes.aT, 0) * args.K * args.N" }] },
"tunableSpace": { "PORTABLE_TILE_K": [16, 32, 64] }
},
{
"name": "bnb4-nf4-speculative-m15-k4096-n4096-b64-band",
"provenance": {
"notes": "Fifteen rows represent a speculative decoder's verification width and exercise the tiled route immediately below M=16. `nf4-tiled-speculative-band-m16-control` brackets that boundary."
},
"preset": "model",
"vars": { "M": 15, "K": 4096, "N": 4096, "blockSize": 64 },
"attrs": { "K": 4096, "N": 4096, "block_size": 64, "quant_type": 1 },
"inputs": {
"aT": { "shape": [15, 4096], "dtype": "float32", "dist": "normal", "seed": 426, "scale": 0.2 },
"bT": { "shape": [8388608], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 },
"absmaxT": {
"shape": [262144],
"dtype": "float32",
"dist": "uniform",
"seed": 427,
"offset": 0.04,
"scale": 0.01,
"signed": false
}
},
"outputs": { "yT": { "shape": [15, 4096], "dtype": "float32" } },
"bench": { "metrics": [{ "type": "gflops", "value": "2 * dim(shapes.aT, 0) * args.K * args.N" }] }
},
{
"name": "bnb4-nf4-speculative-m16-k4096-n4096-b64-tile-floor-control",
"preset": "model",
"vars": { "M": 16, "K": 4096, "N": 4096, "blockSize": 64 },
"attrs": { "K": 4096, "N": 4096, "block_size": 64, "quant_type": 1 },
"inputs": {
"aT": { "shape": [16, 4096], "dtype": "float32", "dist": "normal", "seed": 428, "scale": 0.2 },
"bT": { "shape": [8388608], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 },
"absmaxT": {
"shape": [262144],
"dtype": "float32",
"dist": "uniform",
"seed": 429,
"offset": 0.04,
"scale": 0.01,
"signed": false
}
},
"outputs": { "yT": { "shape": [16, 4096], "dtype": "float32" } },
"bench": { "metrics": [{ "type": "gflops", "value": "2 * dim(shapes.aT, 0) * args.K * args.N" }] }
},
{
"name": "bnb4-nf4-prefill-m64-k4096-n4096-b64-control",
"preset": "model",
"vars": { "M": 64, "K": 4096, "N": 4096, "blockSize": 64 },
"attrs": { "K": 4096, "N": 4096, "block_size": 64, "quant_type": 1 },
"inputs": {
"aT": { "shape": [64, 4096], "dtype": "float32", "dist": "normal", "seed": 430, "scale": 0.2 },
"bT": { "shape": [8388608], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 },
"absmaxT": {
"shape": [262144],
"dtype": "float32",
"dist": "uniform",
"seed": 431,
"offset": 0.04,
"scale": 0.01,
"signed": false
}
},
"outputs": { "yT": { "shape": [64, 4096], "dtype": "float32" } },
"bench": { "metrics": [{ "type": "gflops", "value": "2 * dim(shapes.aT, 0) * args.K * args.N" }] }
},
{
"name": "bnb4-probe-float32-q1-m8-k4096-n4096-b64",
"preset": "stress",
"attrs": { "K": 4096, "N": 4096, "block_size": 64, "quant_type": 1 },
"inputs": {
"aT": { "shape": [8, 4096], "dtype": "float32", "dist": "normal", "seed": 426, "scale": 0.2 },
"bT": { "shape": [8388608], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 },
"absmaxT": { "shape": [262144], "dtype": "float32", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 }
},
"outputs": { "yT": { "shape": [8, 4096], "dtype": "float32" } },
"bench": { "metrics": [{ "type": "gflops", "value": 268435456 }] }
},
{
"name": "bnb4-probe-float32-q1-m32-k4096-n4096-b64",
"preset": "stress",
"attrs": { "K": 4096, "N": 4096, "block_size": 64, "quant_type": 1 },
"inputs": {
"aT": { "shape": [32, 4096], "dtype": "float32", "dist": "normal", "seed": 426, "scale": 0.2 },
"bT": { "shape": [8388608], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 },
"absmaxT": { "shape": [262144], "dtype": "float32", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 }
},
"outputs": { "yT": { "shape": [32, 4096], "dtype": "float32" } },
"bench": { "metrics": [{ "type": "gflops", "value": 1073741824 }] }
},
{
"name": "bnb4-probe-float32-q1-m33-k4096-n4096-b64",
"preset": "stress",
"attrs": { "K": 4096, "N": 4096, "block_size": 64, "quant_type": 1 },
"inputs": {
"aT": { "shape": [33, 4096], "dtype": "float32", "dist": "normal", "seed": 426, "scale": 0.2 },
"bT": { "shape": [8388608], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 },
"absmaxT": { "shape": [262144], "dtype": "float32", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 }
},
"outputs": { "yT": { "shape": [33, 4096], "dtype": "float32" } },
"bench": { "metrics": [{ "type": "gflops", "value": 1107296256 }] }
},
{
"name": "bnb4-probe-float32-q1-m65-k4096-n4096-b64",
"preset": "stress",
"attrs": { "K": 4096, "N": 4096, "block_size": 64, "quant_type": 1 },
"inputs": {
"aT": { "shape": [65, 4096], "dtype": "float32", "dist": "normal", "seed": 426, "scale": 0.2 },
"bT": { "shape": [8388608], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 },
"absmaxT": { "shape": [262144], "dtype": "float32", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 }
},
"outputs": { "yT": { "shape": [65, 4096], "dtype": "float32" } },
"bench": { "metrics": [{ "type": "gflops", "value": 2181038080 }] }
},
{
"name": "bnb4-probe-float16-q1-m15-k4096-n4096-b64",
"preset": "stress",
"attrs": { "K": 4096, "N": 4096, "block_size": 64, "quant_type": 1 },
"inputs": {
"aT": { "shape": [15, 4096], "dtype": "float16", "dist": "normal", "seed": 426, "scale": 0.2 },
"bT": { "shape": [8388608], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 },
"absmaxT": { "shape": [262144], "dtype": "float16", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 }
},
"outputs": { "yT": { "shape": [15, 4096], "dtype": "float16" } },
"bench": { "metrics": [{ "type": "gflops", "value": 503316480 }] }
},
{
"name": "bnb4-probe-float16-q1-m16-k4096-n4096-b64",
"preset": "stress",
"attrs": { "K": 4096, "N": 4096, "block_size": 64, "quant_type": 1 },
"inputs": {
"aT": { "shape": [16, 4096], "dtype": "float16", "dist": "normal", "seed": 426, "scale": 0.2 },
"bT": { "shape": [8388608], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 },
"absmaxT": { "shape": [262144], "dtype": "float16", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 }
},
"outputs": { "yT": { "shape": [16, 4096], "dtype": "float16" } },
"bench": { "metrics": [{ "type": "gflops", "value": 536870912 }] }
},
{
"name": "bnb4-probe-float16-q1-m33-k4096-n4096-b64",
"preset": "stress",
"attrs": { "K": 4096, "N": 4096, "block_size": 64, "quant_type": 1 },
"inputs": {
"aT": { "shape": [33, 4096], "dtype": "float16", "dist": "normal", "seed": 426, "scale": 0.2 },
"bT": { "shape": [8388608], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 },
"absmaxT": { "shape": [262144], "dtype": "float16", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 }
},
"outputs": { "yT": { "shape": [33, 4096], "dtype": "float16" } },
"bench": { "metrics": [{ "type": "gflops", "value": 1107296256 }] }
},
{
"name": "bnb4-probe-float32-q0-m16-k4096-n4096-b64",
"preset": "stress",
"attrs": { "K": 4096, "N": 4096, "block_size": 64, "quant_type": 0 },
"inputs": {
"aT": { "shape": [16, 4096], "dtype": "float32", "dist": "normal", "seed": 426, "scale": 0.2 },
"bT": { "shape": [8388608], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 },
"absmaxT": { "shape": [262144], "dtype": "float32", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 }
},
"outputs": { "yT": { "shape": [16, 4096], "dtype": "float32" } },
"bench": { "metrics": [{ "type": "gflops", "value": 536870912 }] }
},
{
"name": "bnb4-probe-float32-q1-m15-k1024-n128-b64",
"preset": "stress",
"attrs": { "K": 1024, "N": 128, "block_size": 64, "quant_type": 1 },
"inputs": {
"aT": { "shape": [15, 1024], "dtype": "float32", "dist": "normal", "seed": 426, "scale": 0.2 },
"bT": { "shape": [65536], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 },
"absmaxT": { "shape": [2048], "dtype": "float32", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 }
},
"outputs": { "yT": { "shape": [15, 128], "dtype": "float32" } },
"bench": { "metrics": [{ "type": "gflops", "value": 3932160 }] }
},
{
"name": "bnb4-probe-float16-q1-m16-k1024-n128-b64",
"preset": "stress",
"attrs": { "K": 1024, "N": 128, "block_size": 64, "quant_type": 1 },
"inputs": {
"aT": { "shape": [16, 1024], "dtype": "float16", "dist": "normal", "seed": 426, "scale": 0.2 },
"bT": { "shape": [65536], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 },
"absmaxT": { "shape": [2048], "dtype": "float16", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 }
},
"outputs": { "yT": { "shape": [16, 128], "dtype": "float16" } },
"bench": { "metrics": [{ "type": "gflops", "value": 4194304 }] }
},
{
"name": "bnb4-probe-float32-q1-m15-k1023-n257-b64",
"preset": "stress",
"attrs": { "K": 1023, "N": 257, "block_size": 64, "quant_type": 1 },
"inputs": {
"aT": { "shape": [15, 1023], "dtype": "float32", "dist": "normal", "seed": 426, "scale": 0.2 },
"bT": { "shape": [131456], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 },
"absmaxT": { "shape": [4108], "dtype": "float32", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 }
},
"outputs": { "yT": { "shape": [15, 257], "dtype": "float32" } },
"bench": { "metrics": [{ "type": "gflops", "value": 7887330 }] }
},
{
"name": "bnb4-probe-float16-q1-m16-k1023-n257-b64",
"preset": "stress",
"attrs": { "K": 1023, "N": 257, "block_size": 64, "quant_type": 1 },
"inputs": {
"aT": { "shape": [16, 1023], "dtype": "float16", "dist": "normal", "seed": 426, "scale": 0.2 },
"bT": { "shape": [131456], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 },
"absmaxT": { "shape": [4108], "dtype": "float16", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 }
},
"outputs": { "yT": { "shape": [16, 257], "dtype": "float16" } },
"bench": { "metrics": [{ "type": "gflops", "value": 8413152 }] }
},
{
"name": "bnb4-probe-float32-q1-m3-k4096-n4096-b64",
"preset": "stress",
"attrs": { "K": 4096, "N": 4096, "block_size": 64, "quant_type": 1 },
"inputs": {
"aT": { "shape": [3, 4096], "dtype": "float32", "dist": "normal", "seed": 426, "scale": 0.2 },
"bT": { "shape": [8388608], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 },
"absmaxT": { "shape": [262144], "dtype": "float32", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 }
},
"outputs": { "yT": { "shape": [3, 4096], "dtype": "float32" } },
"bench": { "metrics": [{ "type": "gflops", "value": 100663296 }] }
},
{
"name": "bnb4-probe-float32-q1-m129-k1024-n1024-b64",
"preset": "stress",
"attrs": { "K": 1024, "N": 1024, "block_size": 64, "quant_type": 1 },
"inputs": {
"aT": { "shape": [129, 1024], "dtype": "float32", "dist": "normal", "seed": 426, "scale": 0.2 },
"bT": { "shape": [524288], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 },
"absmaxT": { "shape": [16384], "dtype": "float32", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 }
},
"outputs": { "yT": { "shape": [129, 1024], "dtype": "float32" } },
"bench": { "metrics": [{ "type": "gflops", "value": 270532608 }] }
},
{
"name": "bnb4-probe-float16-q1-m191-k1024-n1024-b64",
"preset": "stress",
"attrs": { "K": 1024, "N": 1024, "block_size": 64, "quant_type": 1 },
"inputs": {
"aT": { "shape": [191, 1024], "dtype": "float16", "dist": "normal", "seed": 426, "scale": 0.2 },
"bT": { "shape": [524288], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 },
"absmaxT": { "shape": [16384], "dtype": "float16", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 }
},
"outputs": { "yT": { "shape": [191, 1024], "dtype": "float16" } },
"bench": { "metrics": [{ "type": "gflops", "value": 400556032 }] }
},
{
"name": "bnb4-probe-float16-q1-m255-k1024-n1024-b64",
"preset": "stress",
"attrs": { "K": 1024, "N": 1024, "block_size": 64, "quant_type": 1 },
"inputs": {
"aT": { "shape": [255, 1024], "dtype": "float16", "dist": "normal", "seed": 426, "scale": 0.2 },
"bT": { "shape": [524288], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 },
"absmaxT": { "shape": [16384], "dtype": "float16", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 }
},
"outputs": { "yT": { "shape": [255, 1024], "dtype": "float16" } },
"bench": { "metrics": [{ "type": "gflops", "value": 534773760 }] }
},
{
"name": "bnb4-probe-float32-q1-m255-k4096-n4096-b64",
"preset": "stress",
"attrs": { "K": 4096, "N": 4096, "block_size": 64, "quant_type": 1 },
"inputs": {
"aT": { "shape": [255, 4096], "dtype": "float32", "dist": "normal", "seed": 426, "scale": 0.2 },
"bT": { "shape": [8388608], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 },
"absmaxT": { "shape": [262144], "dtype": "float32", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 }
},
"outputs": { "yT": { "shape": [255, 4096], "dtype": "float32" } },
"bench": { "metrics": [{ "type": "gflops", "value": 8556380160 }] }
},
{
"name": "bnb4-probe-float16-q1-m511-k4096-n4096-b64",
"preset": "stress",
"attrs": { "K": 4096, "N": 4096, "block_size": 64, "quant_type": 1 },
"inputs": {
"aT": { "shape": [511, 4096], "dtype": "float16", "dist": "normal", "seed": 426, "scale": 0.2 },
"bT": { "shape": [8388608], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 },
"absmaxT": { "shape": [262144], "dtype": "float16", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 }
},
"outputs": { "yT": { "shape": [511, 4096], "dtype": "float16" } },
"bench": { "metrics": [{ "type": "gflops", "value": 17146314752 }] }
}
]
}
|