| { |
| "cases": [ |
| { |
| "name": "bnb4-nf4-decode-m1-k2048-n2048-b64", |
| "preset": "smoke", |
| "vars": { "M": 1, "K": 2048, "N": 2048, "blockSize": 64 }, |
| "attrs": { "K": 2048, "N": 2048, "block_size": 64, "quant_type": 1 }, |
| "inputs": { |
| "aT": { "shape": [1, 2048], "dtype": "float32", "dist": "normal", "seed": 410, "scale": 0.2 }, |
| "bT": { "shape": [2097152], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 }, |
| "absmaxT": { |
| "shape": [65536], |
| "dtype": "float32", |
| "dist": "uniform", |
| "seed": 411, |
| "offset": 0.04, |
| "scale": 0.01, |
| "signed": false |
| } |
| }, |
| "outputs": { "yT": { "shape": [1, 2048], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { |
| "type": "bandwidth", |
| "value": "numel(shapes.bT) * 4 + (numel(shapes.aT) + numel(shapes.absmaxT) + numel(shapes.yT)) * 4" |
| } |
| ] |
| } |
| }, |
| { |
| "name": "bnb4-nf4-prefill-m128-k2048-n2048-b64", |
| "preset": "smoke", |
| "vars": { "M": 128, "K": 2048, "N": 2048, "blockSize": 64 }, |
| "attrs": { "K": 2048, "N": 2048, "block_size": 64, "quant_type": 1 }, |
| "tunableSpace": { "PORTABLE_TILE_K": [16, 32, 64] }, |
| "inputs": { |
| "aT": { "shape": [128, 2048], "dtype": "float32", "dist": "normal", "seed": 412, "scale": 0.2 }, |
| "bT": { "shape": [2097152], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 }, |
| "absmaxT": { |
| "shape": [65536], |
| "dtype": "float32", |
| "dist": "uniform", |
| "seed": 413, |
| "offset": 0.04, |
| "scale": 0.01, |
| "signed": false |
| } |
| }, |
| "outputs": { "yT": { "shape": [128, 2048], "dtype": "float32" } }, |
| "bench": { |
| "primary": true, |
| "metrics": [{ "type": "gflops", "value": "2 * dim(shapes.aT, 0) * args.K * args.N" }] |
| } |
| }, |
| { |
| "name": "bnb4-fp4-prefill-m128-k2048-n2048-b64", |
| "preset": "smoke", |
| "vars": { "M": 128, "K": 2048, "N": 2048, "blockSize": 64 }, |
| "attrs": { "K": 2048, "N": 2048, "block_size": 64, "quant_type": 0 }, |
| "tunableSpace": { "PORTABLE_TILE_K": [16, 32, 64] }, |
| "inputs": { |
| "aT": { "shape": [128, 2048], "dtype": "float32", "dist": "normal", "seed": 414, "scale": 0.2 }, |
| "bT": { "shape": [2097152], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 }, |
| "absmaxT": { |
| "shape": [65536], |
| "dtype": "float32", |
| "dist": "uniform", |
| "seed": 415, |
| "offset": 0.04, |
| "scale": 0.01, |
| "signed": false |
| } |
| }, |
| "outputs": { "yT": { "shape": [128, 2048], "dtype": "float32" } }, |
| "bench": { "metrics": [{ "type": "gflops", "value": "2 * dim(shapes.aT, 0) * args.K * args.N" }] } |
| }, |
| { |
| "name": "bnb4-nf4-prefill-m512-k4096-n4096-b64", |
| "preset": "model", |
| "vars": { "M": 512, "K": 4096, "N": 4096, "blockSize": 64 }, |
| "attrs": { "K": 4096, "N": 4096, "block_size": 64, "quant_type": 1 }, |
| "inputs": { |
| "aT": { "shape": [512, 4096], "dtype": "float32", "dist": "normal", "seed": 416, "scale": 0.2 }, |
| "bT": { "shape": [8388608], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 }, |
| "absmaxT": { |
| "shape": [262144], |
| "dtype": "float32", |
| "dist": "uniform", |
| "seed": 417, |
| "offset": 0.04, |
| "scale": 0.01, |
| "signed": false |
| } |
| }, |
| "outputs": { "yT": { "shape": [512, 4096], "dtype": "float32" } }, |
| "bench": { "metrics": [{ "type": "gflops", "value": "2 * dim(shapes.aT, 0) * args.K * args.N" }] }, |
| "tunableSpace": { "PORTABLE_TILE_K": [16, 32, 64] } |
| }, |
| { |
| "name": "bnb4-nf4-prefill-m512-k4096-n4096-b64-f16", |
| "preset": "model", |
| "vars": { "M": 512, "K": 4096, "N": 4096, "blockSize": 64 }, |
| "attrs": { "K": 4096, "N": 4096, "block_size": 64, "quant_type": 1 }, |
| "inputs": { |
| "aT": { "shape": [512, 4096], "dtype": "float16", "dist": "normal", "seed": 416, "scale": 0.2 }, |
| "bT": { "shape": [8388608], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 }, |
| "absmaxT": { |
| "shape": [262144], |
| "dtype": "float16", |
| "dist": "uniform", |
| "seed": 417, |
| "offset": 0.04, |
| "scale": 0.01, |
| "signed": false |
| } |
| }, |
| "outputs": { "yT": { "shape": [512, 4096], "dtype": "float16" } }, |
| "bench": { "metrics": [{ "type": "gflops", "value": "2 * dim(shapes.aT, 0) * args.K * args.N" }] }, |
| "tunableSpace": { "PORTABLE_TILE_K": [16, 32, 64] } |
| }, |
| { |
| "name": "bnb4-nf4-speculative-m15-k4096-n4096-b64-band", |
| "provenance": { |
| "notes": "Fifteen rows represent a speculative decoder's verification width and exercise the tiled route immediately below M=16. `nf4-tiled-speculative-band-m16-control` brackets that boundary." |
| }, |
| "preset": "model", |
| "vars": { "M": 15, "K": 4096, "N": 4096, "blockSize": 64 }, |
| "attrs": { "K": 4096, "N": 4096, "block_size": 64, "quant_type": 1 }, |
| "inputs": { |
| "aT": { "shape": [15, 4096], "dtype": "float32", "dist": "normal", "seed": 426, "scale": 0.2 }, |
| "bT": { "shape": [8388608], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 }, |
| "absmaxT": { |
| "shape": [262144], |
| "dtype": "float32", |
| "dist": "uniform", |
| "seed": 427, |
| "offset": 0.04, |
| "scale": 0.01, |
| "signed": false |
| } |
| }, |
| "outputs": { "yT": { "shape": [15, 4096], "dtype": "float32" } }, |
| "bench": { "metrics": [{ "type": "gflops", "value": "2 * dim(shapes.aT, 0) * args.K * args.N" }] } |
| }, |
| { |
| "name": "bnb4-nf4-speculative-m16-k4096-n4096-b64-tile-floor-control", |
| "preset": "model", |
| "vars": { "M": 16, "K": 4096, "N": 4096, "blockSize": 64 }, |
| "attrs": { "K": 4096, "N": 4096, "block_size": 64, "quant_type": 1 }, |
| "inputs": { |
| "aT": { "shape": [16, 4096], "dtype": "float32", "dist": "normal", "seed": 428, "scale": 0.2 }, |
| "bT": { "shape": [8388608], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 }, |
| "absmaxT": { |
| "shape": [262144], |
| "dtype": "float32", |
| "dist": "uniform", |
| "seed": 429, |
| "offset": 0.04, |
| "scale": 0.01, |
| "signed": false |
| } |
| }, |
| "outputs": { "yT": { "shape": [16, 4096], "dtype": "float32" } }, |
| "bench": { "metrics": [{ "type": "gflops", "value": "2 * dim(shapes.aT, 0) * args.K * args.N" }] } |
| }, |
| { |
| "name": "bnb4-nf4-prefill-m64-k4096-n4096-b64-control", |
| "preset": "model", |
| "vars": { "M": 64, "K": 4096, "N": 4096, "blockSize": 64 }, |
| "attrs": { "K": 4096, "N": 4096, "block_size": 64, "quant_type": 1 }, |
| "inputs": { |
| "aT": { "shape": [64, 4096], "dtype": "float32", "dist": "normal", "seed": 430, "scale": 0.2 }, |
| "bT": { "shape": [8388608], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 }, |
| "absmaxT": { |
| "shape": [262144], |
| "dtype": "float32", |
| "dist": "uniform", |
| "seed": 431, |
| "offset": 0.04, |
| "scale": 0.01, |
| "signed": false |
| } |
| }, |
| "outputs": { "yT": { "shape": [64, 4096], "dtype": "float32" } }, |
| "bench": { "metrics": [{ "type": "gflops", "value": "2 * dim(shapes.aT, 0) * args.K * args.N" }] } |
| }, |
| { |
| "name": "bnb4-probe-float32-q1-m8-k4096-n4096-b64", |
| "preset": "stress", |
| "attrs": { "K": 4096, "N": 4096, "block_size": 64, "quant_type": 1 }, |
| "inputs": { |
| "aT": { "shape": [8, 4096], "dtype": "float32", "dist": "normal", "seed": 426, "scale": 0.2 }, |
| "bT": { "shape": [8388608], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 }, |
| "absmaxT": { "shape": [262144], "dtype": "float32", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 } |
| }, |
| "outputs": { "yT": { "shape": [8, 4096], "dtype": "float32" } }, |
| "bench": { "metrics": [{ "type": "gflops", "value": 268435456 }] } |
| }, |
| { |
| "name": "bnb4-probe-float32-q1-m32-k4096-n4096-b64", |
| "preset": "stress", |
| "attrs": { "K": 4096, "N": 4096, "block_size": 64, "quant_type": 1 }, |
| "inputs": { |
| "aT": { "shape": [32, 4096], "dtype": "float32", "dist": "normal", "seed": 426, "scale": 0.2 }, |
| "bT": { "shape": [8388608], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 }, |
| "absmaxT": { "shape": [262144], "dtype": "float32", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 } |
| }, |
| "outputs": { "yT": { "shape": [32, 4096], "dtype": "float32" } }, |
| "bench": { "metrics": [{ "type": "gflops", "value": 1073741824 }] } |
| }, |
| { |
| "name": "bnb4-probe-float32-q1-m33-k4096-n4096-b64", |
| "preset": "stress", |
| "attrs": { "K": 4096, "N": 4096, "block_size": 64, "quant_type": 1 }, |
| "inputs": { |
| "aT": { "shape": [33, 4096], "dtype": "float32", "dist": "normal", "seed": 426, "scale": 0.2 }, |
| "bT": { "shape": [8388608], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 }, |
| "absmaxT": { "shape": [262144], "dtype": "float32", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 } |
| }, |
| "outputs": { "yT": { "shape": [33, 4096], "dtype": "float32" } }, |
| "bench": { "metrics": [{ "type": "gflops", "value": 1107296256 }] } |
| }, |
| { |
| "name": "bnb4-probe-float32-q1-m65-k4096-n4096-b64", |
| "preset": "stress", |
| "attrs": { "K": 4096, "N": 4096, "block_size": 64, "quant_type": 1 }, |
| "inputs": { |
| "aT": { "shape": [65, 4096], "dtype": "float32", "dist": "normal", "seed": 426, "scale": 0.2 }, |
| "bT": { "shape": [8388608], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 }, |
| "absmaxT": { "shape": [262144], "dtype": "float32", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 } |
| }, |
| "outputs": { "yT": { "shape": [65, 4096], "dtype": "float32" } }, |
| "bench": { "metrics": [{ "type": "gflops", "value": 2181038080 }] } |
| }, |
| { |
| "name": "bnb4-probe-float16-q1-m15-k4096-n4096-b64", |
| "preset": "stress", |
| "attrs": { "K": 4096, "N": 4096, "block_size": 64, "quant_type": 1 }, |
| "inputs": { |
| "aT": { "shape": [15, 4096], "dtype": "float16", "dist": "normal", "seed": 426, "scale": 0.2 }, |
| "bT": { "shape": [8388608], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 }, |
| "absmaxT": { "shape": [262144], "dtype": "float16", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 } |
| }, |
| "outputs": { "yT": { "shape": [15, 4096], "dtype": "float16" } }, |
| "bench": { "metrics": [{ "type": "gflops", "value": 503316480 }] } |
| }, |
| { |
| "name": "bnb4-probe-float16-q1-m16-k4096-n4096-b64", |
| "preset": "stress", |
| "attrs": { "K": 4096, "N": 4096, "block_size": 64, "quant_type": 1 }, |
| "inputs": { |
| "aT": { "shape": [16, 4096], "dtype": "float16", "dist": "normal", "seed": 426, "scale": 0.2 }, |
| "bT": { "shape": [8388608], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 }, |
| "absmaxT": { "shape": [262144], "dtype": "float16", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 } |
| }, |
| "outputs": { "yT": { "shape": [16, 4096], "dtype": "float16" } }, |
| "bench": { "metrics": [{ "type": "gflops", "value": 536870912 }] } |
| }, |
| { |
| "name": "bnb4-probe-float16-q1-m33-k4096-n4096-b64", |
| "preset": "stress", |
| "attrs": { "K": 4096, "N": 4096, "block_size": 64, "quant_type": 1 }, |
| "inputs": { |
| "aT": { "shape": [33, 4096], "dtype": "float16", "dist": "normal", "seed": 426, "scale": 0.2 }, |
| "bT": { "shape": [8388608], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 }, |
| "absmaxT": { "shape": [262144], "dtype": "float16", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 } |
| }, |
| "outputs": { "yT": { "shape": [33, 4096], "dtype": "float16" } }, |
| "bench": { "metrics": [{ "type": "gflops", "value": 1107296256 }] } |
| }, |
| { |
| "name": "bnb4-probe-float32-q0-m16-k4096-n4096-b64", |
| "preset": "stress", |
| "attrs": { "K": 4096, "N": 4096, "block_size": 64, "quant_type": 0 }, |
| "inputs": { |
| "aT": { "shape": [16, 4096], "dtype": "float32", "dist": "normal", "seed": 426, "scale": 0.2 }, |
| "bT": { "shape": [8388608], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 }, |
| "absmaxT": { "shape": [262144], "dtype": "float32", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 } |
| }, |
| "outputs": { "yT": { "shape": [16, 4096], "dtype": "float32" } }, |
| "bench": { "metrics": [{ "type": "gflops", "value": 536870912 }] } |
| }, |
| { |
| "name": "bnb4-probe-float32-q1-m15-k1024-n128-b64", |
| "preset": "stress", |
| "attrs": { "K": 1024, "N": 128, "block_size": 64, "quant_type": 1 }, |
| "inputs": { |
| "aT": { "shape": [15, 1024], "dtype": "float32", "dist": "normal", "seed": 426, "scale": 0.2 }, |
| "bT": { "shape": [65536], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 }, |
| "absmaxT": { "shape": [2048], "dtype": "float32", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 } |
| }, |
| "outputs": { "yT": { "shape": [15, 128], "dtype": "float32" } }, |
| "bench": { "metrics": [{ "type": "gflops", "value": 3932160 }] } |
| }, |
| { |
| "name": "bnb4-probe-float16-q1-m16-k1024-n128-b64", |
| "preset": "stress", |
| "attrs": { "K": 1024, "N": 128, "block_size": 64, "quant_type": 1 }, |
| "inputs": { |
| "aT": { "shape": [16, 1024], "dtype": "float16", "dist": "normal", "seed": 426, "scale": 0.2 }, |
| "bT": { "shape": [65536], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 }, |
| "absmaxT": { "shape": [2048], "dtype": "float16", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 } |
| }, |
| "outputs": { "yT": { "shape": [16, 128], "dtype": "float16" } }, |
| "bench": { "metrics": [{ "type": "gflops", "value": 4194304 }] } |
| }, |
| { |
| "name": "bnb4-probe-float32-q1-m15-k1023-n257-b64", |
| "preset": "stress", |
| "attrs": { "K": 1023, "N": 257, "block_size": 64, "quant_type": 1 }, |
| "inputs": { |
| "aT": { "shape": [15, 1023], "dtype": "float32", "dist": "normal", "seed": 426, "scale": 0.2 }, |
| "bT": { "shape": [131456], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 }, |
| "absmaxT": { "shape": [4108], "dtype": "float32", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 } |
| }, |
| "outputs": { "yT": { "shape": [15, 257], "dtype": "float32" } }, |
| "bench": { "metrics": [{ "type": "gflops", "value": 7887330 }] } |
| }, |
| { |
| "name": "bnb4-probe-float16-q1-m16-k1023-n257-b64", |
| "preset": "stress", |
| "attrs": { "K": 1023, "N": 257, "block_size": 64, "quant_type": 1 }, |
| "inputs": { |
| "aT": { "shape": [16, 1023], "dtype": "float16", "dist": "normal", "seed": 426, "scale": 0.2 }, |
| "bT": { "shape": [131456], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 }, |
| "absmaxT": { "shape": [4108], "dtype": "float16", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 } |
| }, |
| "outputs": { "yT": { "shape": [16, 257], "dtype": "float16" } }, |
| "bench": { "metrics": [{ "type": "gflops", "value": 8413152 }] } |
| }, |
| { |
| "name": "bnb4-probe-float32-q1-m3-k4096-n4096-b64", |
| "preset": "stress", |
| "attrs": { "K": 4096, "N": 4096, "block_size": 64, "quant_type": 1 }, |
| "inputs": { |
| "aT": { "shape": [3, 4096], "dtype": "float32", "dist": "normal", "seed": 426, "scale": 0.2 }, |
| "bT": { "shape": [8388608], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 }, |
| "absmaxT": { "shape": [262144], "dtype": "float32", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 } |
| }, |
| "outputs": { "yT": { "shape": [3, 4096], "dtype": "float32" } }, |
| "bench": { "metrics": [{ "type": "gflops", "value": 100663296 }] } |
| }, |
| { |
| "name": "bnb4-probe-float32-q1-m129-k1024-n1024-b64", |
| "preset": "stress", |
| "attrs": { "K": 1024, "N": 1024, "block_size": 64, "quant_type": 1 }, |
| "inputs": { |
| "aT": { "shape": [129, 1024], "dtype": "float32", "dist": "normal", "seed": 426, "scale": 0.2 }, |
| "bT": { "shape": [524288], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 }, |
| "absmaxT": { "shape": [16384], "dtype": "float32", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 } |
| }, |
| "outputs": { "yT": { "shape": [129, 1024], "dtype": "float32" } }, |
| "bench": { "metrics": [{ "type": "gflops", "value": 270532608 }] } |
| }, |
| { |
| "name": "bnb4-probe-float16-q1-m191-k1024-n1024-b64", |
| "preset": "stress", |
| "attrs": { "K": 1024, "N": 1024, "block_size": 64, "quant_type": 1 }, |
| "inputs": { |
| "aT": { "shape": [191, 1024], "dtype": "float16", "dist": "normal", "seed": 426, "scale": 0.2 }, |
| "bT": { "shape": [524288], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 }, |
| "absmaxT": { "shape": [16384], "dtype": "float16", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 } |
| }, |
| "outputs": { "yT": { "shape": [191, 1024], "dtype": "float16" } }, |
| "bench": { "metrics": [{ "type": "gflops", "value": 400556032 }] } |
| }, |
| { |
| "name": "bnb4-probe-float16-q1-m255-k1024-n1024-b64", |
| "preset": "stress", |
| "attrs": { "K": 1024, "N": 1024, "block_size": 64, "quant_type": 1 }, |
| "inputs": { |
| "aT": { "shape": [255, 1024], "dtype": "float16", "dist": "normal", "seed": 426, "scale": 0.2 }, |
| "bT": { "shape": [524288], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 }, |
| "absmaxT": { "shape": [16384], "dtype": "float16", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 } |
| }, |
| "outputs": { "yT": { "shape": [255, 1024], "dtype": "float16" } }, |
| "bench": { "metrics": [{ "type": "gflops", "value": 534773760 }] } |
| }, |
| { |
| "name": "bnb4-probe-float32-q1-m255-k4096-n4096-b64", |
| "preset": "stress", |
| "attrs": { "K": 4096, "N": 4096, "block_size": 64, "quant_type": 1 }, |
| "inputs": { |
| "aT": { "shape": [255, 4096], "dtype": "float32", "dist": "normal", "seed": 426, "scale": 0.2 }, |
| "bT": { "shape": [8388608], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 }, |
| "absmaxT": { "shape": [262144], "dtype": "float32", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 } |
| }, |
| "outputs": { "yT": { "shape": [255, 4096], "dtype": "float32" } }, |
| "bench": { "metrics": [{ "type": "gflops", "value": 8556380160 }] } |
| }, |
| { |
| "name": "bnb4-probe-float16-q1-m511-k4096-n4096-b64", |
| "preset": "stress", |
| "attrs": { "K": 4096, "N": 4096, "block_size": 64, "quant_type": 1 }, |
| "inputs": { |
| "aT": { "shape": [511, 4096], "dtype": "float16", "dist": "normal", "seed": 426, "scale": 0.2 }, |
| "bT": { "shape": [8388608], "dtype": "uint8", "dist": "linearMod", "mod": 256, "step": 7 }, |
| "absmaxT": { "shape": [262144], "dtype": "float16", "dist": "uniform", "seed": 427, "min": 0.04, "max": 0.05 } |
| }, |
| "outputs": { "yT": { "shape": [511, 4096], "dtype": "float16" } }, |
| "bench": { "metrics": [{ "type": "gflops", "value": 17146314752 }] } |
| } |
| ] |
| } |
|
|
