| { |
| "metadata": { |
| "ParamSize": 325, |
| "ParamBytes": 2149644288.0, |
| "BitsPerParam": 4.500600961055312 |
| }, |
| "records": [ |
| { |
| "dataPath": "params_shard_0.bin", |
| "format": "raw-shard", |
| "nbytes": 49250304, |
| "records": [ |
| { |
| "name": "lm_head.q_weight", |
| "shape": [ |
| 32064, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 49250304, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "8b7dc0e29e1c572c3c8c79f3545116a6" |
| }, |
| { |
| "dataPath": "params_shard_1.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.21.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "291fa3d154d803e65ea132a1a148ba94" |
| }, |
| { |
| "dataPath": "params_shard_2.bin", |
| "format": "raw-shard", |
| "nbytes": 23470080, |
| "records": [ |
| { |
| "name": "lm_head.q_scale", |
| "shape": [ |
| 32064, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6156288, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.21.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 6156288 |
| }, |
| { |
| "name": "transformer.h.21.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 6162432 |
| }, |
| { |
| "name": "transformer.h.21.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 18745344 |
| }, |
| { |
| "name": "transformer.h.21.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 20318208 |
| }, |
| { |
| "name": "transformer.h.21.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 23463936 |
| } |
| ], |
| "md5sum": "2232562712a502bb30428199cd9d240f" |
| }, |
| { |
| "dataPath": "params_shard_3.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.22.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "6aa51bc82e13a0a09d2bd79ecf706f56" |
| }, |
| { |
| "dataPath": "params_shard_4.bin", |
| "format": "raw-shard", |
| "nbytes": 33239040, |
| "records": [ |
| { |
| "name": "transformer.h.21.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.21.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.22.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 15925248 |
| }, |
| { |
| "name": "transformer.h.22.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 15931392 |
| }, |
| { |
| "name": "transformer.h.22.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 28514304 |
| }, |
| { |
| "name": "transformer.h.22.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 30087168 |
| }, |
| { |
| "name": "transformer.h.22.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 33232896 |
| } |
| ], |
| "md5sum": "80a32541f0f5e6d18f5bc3e728a8a81f" |
| }, |
| { |
| "dataPath": "params_shard_5.bin", |
| "format": "raw-shard", |
| "nbytes": 21239808, |
| "records": [ |
| { |
| "name": "transformer.h.22.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.22.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 4718592 |
| }, |
| { |
| "name": "transformer.h.22.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 5308416 |
| }, |
| { |
| "name": "transformer.h.22.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 19464192 |
| }, |
| { |
| "name": "transformer.h.23.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 21233664 |
| } |
| ], |
| "md5sum": "2485cde6690b1ca0f11482f1a3fce948" |
| }, |
| { |
| "dataPath": "params_shard_6.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.23.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "038753ac7a61c5c8fc1f37e48a163184" |
| }, |
| { |
| "dataPath": "params_shard_7.bin", |
| "format": "raw-shard", |
| "nbytes": 22616064, |
| "records": [ |
| { |
| "name": "transformer.h.23.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.23.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 12582912 |
| }, |
| { |
| "name": "transformer.h.23.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.23.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 17301504 |
| }, |
| { |
| "name": "transformer.h.23.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 17307648 |
| }, |
| { |
| "name": "transformer.h.23.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 22026240 |
| } |
| ], |
| "md5sum": "7ca7072dc49d39e5028da311c2da59f6" |
| }, |
| { |
| "dataPath": "params_shard_8.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.24.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "10ec010d75596ab4288c0101cdaeaed3" |
| }, |
| { |
| "dataPath": "params_shard_9.bin", |
| "format": "raw-shard", |
| "nbytes": 33239040, |
| "records": [ |
| { |
| "name": "transformer.h.23.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.23.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.24.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 15925248 |
| }, |
| { |
| "name": "transformer.h.24.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 15931392 |
| }, |
| { |
| "name": "transformer.h.24.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 28514304 |
| }, |
| { |
| "name": "transformer.h.24.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 30087168 |
| }, |
| { |
| "name": "transformer.h.24.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 33232896 |
| } |
| ], |
| "md5sum": "f4ab5f8b0b0d589d9b1411243a381d97" |
| }, |
| { |
| "dataPath": "params_shard_10.bin", |
| "format": "raw-shard", |
| "nbytes": 21239808, |
| "records": [ |
| { |
| "name": "transformer.h.24.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.24.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 4718592 |
| }, |
| { |
| "name": "transformer.h.24.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 5308416 |
| }, |
| { |
| "name": "transformer.h.24.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 19464192 |
| }, |
| { |
| "name": "transformer.h.25.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 21233664 |
| } |
| ], |
| "md5sum": "3de4e4bb86177439388236f247893c0c" |
| }, |
| { |
| "dataPath": "params_shard_11.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.25.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "d0e4431ab49bbe7282f00262b88e11ac" |
| }, |
| { |
| "dataPath": "params_shard_12.bin", |
| "format": "raw-shard", |
| "nbytes": 22616064, |
| "records": [ |
| { |
| "name": "transformer.h.25.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.25.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 12582912 |
| }, |
| { |
| "name": "transformer.h.25.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.25.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 17301504 |
| }, |
| { |
| "name": "transformer.h.25.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 17307648 |
| }, |
| { |
| "name": "transformer.h.25.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 22026240 |
| } |
| ], |
| "md5sum": "21ffcb98e9f9e5b7eedbb6e92c661716" |
| }, |
| { |
| "dataPath": "params_shard_13.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.26.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "244eba2a84e4a3bc56632585f826328b" |
| }, |
| { |
| "dataPath": "params_shard_14.bin", |
| "format": "raw-shard", |
| "nbytes": 33239040, |
| "records": [ |
| { |
| "name": "transformer.h.25.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.25.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.26.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 15925248 |
| }, |
| { |
| "name": "transformer.h.26.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 15931392 |
| }, |
| { |
| "name": "transformer.h.26.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 28514304 |
| }, |
| { |
| "name": "transformer.h.26.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 30087168 |
| }, |
| { |
| "name": "transformer.h.26.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 33232896 |
| } |
| ], |
| "md5sum": "2a9f1879dcbdb4843449b6b588089233" |
| }, |
| { |
| "dataPath": "params_shard_15.bin", |
| "format": "raw-shard", |
| "nbytes": 21239808, |
| "records": [ |
| { |
| "name": "transformer.h.26.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.26.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 4718592 |
| }, |
| { |
| "name": "transformer.h.26.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 5308416 |
| }, |
| { |
| "name": "transformer.h.26.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 19464192 |
| }, |
| { |
| "name": "transformer.h.27.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 21233664 |
| } |
| ], |
| "md5sum": "db94e9d2b995403fc11c3fe2573463f4" |
| }, |
| { |
| "dataPath": "params_shard_16.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.27.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "e77dfa662ddf8497df0f796e0dfb7797" |
| }, |
| { |
| "dataPath": "params_shard_17.bin", |
| "format": "raw-shard", |
| "nbytes": 22616064, |
| "records": [ |
| { |
| "name": "transformer.h.27.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.27.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 12582912 |
| }, |
| { |
| "name": "transformer.h.27.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.27.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 17301504 |
| }, |
| { |
| "name": "transformer.h.27.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 17307648 |
| }, |
| { |
| "name": "transformer.h.27.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 22026240 |
| } |
| ], |
| "md5sum": "61062289a20a74ca7acce143db7ff09f" |
| }, |
| { |
| "dataPath": "params_shard_18.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.28.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "468d07c6c0d4decfadcb24eb2384f21f" |
| }, |
| { |
| "dataPath": "params_shard_19.bin", |
| "format": "raw-shard", |
| "nbytes": 33239040, |
| "records": [ |
| { |
| "name": "transformer.h.27.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.27.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.28.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 15925248 |
| }, |
| { |
| "name": "transformer.h.28.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 15931392 |
| }, |
| { |
| "name": "transformer.h.28.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 28514304 |
| }, |
| { |
| "name": "transformer.h.28.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 30087168 |
| }, |
| { |
| "name": "transformer.h.28.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 33232896 |
| } |
| ], |
| "md5sum": "7ea9ba97de223dec60d8d652a4de4f93" |
| }, |
| { |
| "dataPath": "params_shard_20.bin", |
| "format": "raw-shard", |
| "nbytes": 21239808, |
| "records": [ |
| { |
| "name": "transformer.h.28.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.28.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 4718592 |
| }, |
| { |
| "name": "transformer.h.28.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 5308416 |
| }, |
| { |
| "name": "transformer.h.28.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 19464192 |
| }, |
| { |
| "name": "transformer.h.29.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 21233664 |
| } |
| ], |
| "md5sum": "7309354a4d39fe28c714851b250b7f13" |
| }, |
| { |
| "dataPath": "params_shard_21.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.29.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "e516301a253bb0a02a4cfabc742f095a" |
| }, |
| { |
| "dataPath": "params_shard_22.bin", |
| "format": "raw-shard", |
| "nbytes": 22616064, |
| "records": [ |
| { |
| "name": "transformer.h.29.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.29.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 12582912 |
| }, |
| { |
| "name": "transformer.h.29.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.29.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 17301504 |
| }, |
| { |
| "name": "transformer.h.29.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 17307648 |
| }, |
| { |
| "name": "transformer.h.29.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 22026240 |
| } |
| ], |
| "md5sum": "519ceaa2aa175d7cea5af5d4d22894bc" |
| }, |
| { |
| "dataPath": "params_shard_23.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.30.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "b7a3d71b718be0cd079fc4592ae36cee" |
| }, |
| { |
| "dataPath": "params_shard_24.bin", |
| "format": "raw-shard", |
| "nbytes": 33239040, |
| "records": [ |
| { |
| "name": "transformer.h.29.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.29.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.30.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 15925248 |
| }, |
| { |
| "name": "transformer.h.30.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 15931392 |
| }, |
| { |
| "name": "transformer.h.30.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 28514304 |
| }, |
| { |
| "name": "transformer.h.30.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 30087168 |
| }, |
| { |
| "name": "transformer.h.30.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 33232896 |
| } |
| ], |
| "md5sum": "51694630cfe5563d88f7129d486080e1" |
| }, |
| { |
| "dataPath": "params_shard_25.bin", |
| "format": "raw-shard", |
| "nbytes": 21239808, |
| "records": [ |
| { |
| "name": "transformer.h.30.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.30.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 4718592 |
| }, |
| { |
| "name": "transformer.h.30.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 5308416 |
| }, |
| { |
| "name": "transformer.h.30.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 19464192 |
| }, |
| { |
| "name": "transformer.h.31.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 21233664 |
| } |
| ], |
| "md5sum": "52c95231effd3abcc0010bd99f64b661" |
| }, |
| { |
| "dataPath": "params_shard_26.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.31.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "42aec8a789db093d5a850365beca1b9a" |
| }, |
| { |
| "dataPath": "params_shard_27.bin", |
| "format": "raw-shard", |
| "nbytes": 22616064, |
| "records": [ |
| { |
| "name": "transformer.h.31.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.31.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 12582912 |
| }, |
| { |
| "name": "transformer.h.31.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.31.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 17301504 |
| }, |
| { |
| "name": "transformer.h.31.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 17307648 |
| }, |
| { |
| "name": "transformer.h.31.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 22026240 |
| } |
| ], |
| "md5sum": "d0a102a92f5c015bb1e71cdeb632fe0c" |
| }, |
| { |
| "dataPath": "params_shard_28.bin", |
| "format": "raw-shard", |
| "nbytes": 49250304, |
| "records": [ |
| { |
| "name": "transformer.embd.q_weight", |
| "shape": [ |
| 32064, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 49250304, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "e65cd6e13f78175ce2f5fcb3844136e4" |
| }, |
| { |
| "dataPath": "params_shard_29.bin", |
| "format": "raw-shard", |
| "nbytes": 22093824, |
| "records": [ |
| { |
| "name": "transformer.h.31.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.31.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.norm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 15925248 |
| }, |
| { |
| "name": "transformer.embd.q_scale", |
| "shape": [ |
| 32064, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6156288, |
| "byteOffset": 15931392 |
| }, |
| { |
| "name": "transformer.h.0.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 22087680 |
| } |
| ], |
| "md5sum": "13906ed5091c3b4cabe7f7448ea53d40" |
| }, |
| { |
| "dataPath": "params_shard_30.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.0.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "01a5d0a50e64f55fba559fee50cf9fa9" |
| }, |
| { |
| "dataPath": "params_shard_31.bin", |
| "format": "raw-shard", |
| "nbytes": 22616064, |
| "records": [ |
| { |
| "name": "transformer.h.0.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.0.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 12582912 |
| }, |
| { |
| "name": "transformer.h.0.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.0.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 17301504 |
| }, |
| { |
| "name": "transformer.h.0.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 17307648 |
| }, |
| { |
| "name": "transformer.h.0.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 22026240 |
| } |
| ], |
| "md5sum": "629271dbb7fcacc486a2b5191b5457c7" |
| }, |
| { |
| "dataPath": "params_shard_32.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.1.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "06b55aec3bd00e550e033e2c8d6ac0ac" |
| }, |
| { |
| "dataPath": "params_shard_33.bin", |
| "format": "raw-shard", |
| "nbytes": 33239040, |
| "records": [ |
| { |
| "name": "transformer.h.0.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.0.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.1.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 15925248 |
| }, |
| { |
| "name": "transformer.h.1.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 15931392 |
| }, |
| { |
| "name": "transformer.h.1.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 28514304 |
| }, |
| { |
| "name": "transformer.h.1.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 30087168 |
| }, |
| { |
| "name": "transformer.h.1.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 33232896 |
| } |
| ], |
| "md5sum": "0baa80591c549d23d1a9176e7c8d8672" |
| }, |
| { |
| "dataPath": "params_shard_34.bin", |
| "format": "raw-shard", |
| "nbytes": 21239808, |
| "records": [ |
| { |
| "name": "transformer.h.1.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.1.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 4718592 |
| }, |
| { |
| "name": "transformer.h.1.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 5308416 |
| }, |
| { |
| "name": "transformer.h.1.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 19464192 |
| }, |
| { |
| "name": "transformer.h.10.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 21233664 |
| } |
| ], |
| "md5sum": "5584b898584835fcd5480db5f0e4975f" |
| }, |
| { |
| "dataPath": "params_shard_35.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.10.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "34d6d5bb2604bf2c325306582370cc74" |
| }, |
| { |
| "dataPath": "params_shard_36.bin", |
| "format": "raw-shard", |
| "nbytes": 22616064, |
| "records": [ |
| { |
| "name": "transformer.h.10.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.10.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 12582912 |
| }, |
| { |
| "name": "transformer.h.10.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.10.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 17301504 |
| }, |
| { |
| "name": "transformer.h.10.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 17307648 |
| }, |
| { |
| "name": "transformer.h.10.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 22026240 |
| } |
| ], |
| "md5sum": "0dda9f13d0d93ef2639737ef153eba07" |
| }, |
| { |
| "dataPath": "params_shard_37.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.11.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "978d47b4e8ab28da90916bdf8b5fd352" |
| }, |
| { |
| "dataPath": "params_shard_38.bin", |
| "format": "raw-shard", |
| "nbytes": 33239040, |
| "records": [ |
| { |
| "name": "transformer.h.10.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.10.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.11.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 15925248 |
| }, |
| { |
| "name": "transformer.h.11.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 15931392 |
| }, |
| { |
| "name": "transformer.h.11.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 28514304 |
| }, |
| { |
| "name": "transformer.h.11.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 30087168 |
| }, |
| { |
| "name": "transformer.h.11.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 33232896 |
| } |
| ], |
| "md5sum": "44b1c9c79d2b4eee0bc34d275fe175fa" |
| }, |
| { |
| "dataPath": "params_shard_39.bin", |
| "format": "raw-shard", |
| "nbytes": 21239808, |
| "records": [ |
| { |
| "name": "transformer.h.11.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.11.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 4718592 |
| }, |
| { |
| "name": "transformer.h.11.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 5308416 |
| }, |
| { |
| "name": "transformer.h.11.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 19464192 |
| }, |
| { |
| "name": "transformer.h.12.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 21233664 |
| } |
| ], |
| "md5sum": "4666acf541e4ffd237a29411256a97d2" |
| }, |
| { |
| "dataPath": "params_shard_40.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.12.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "e79ed66deccd5e690ccfe389f77031a2" |
| }, |
| { |
| "dataPath": "params_shard_41.bin", |
| "format": "raw-shard", |
| "nbytes": 22616064, |
| "records": [ |
| { |
| "name": "transformer.h.12.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.12.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 12582912 |
| }, |
| { |
| "name": "transformer.h.12.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.12.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 17301504 |
| }, |
| { |
| "name": "transformer.h.12.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 17307648 |
| }, |
| { |
| "name": "transformer.h.12.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 22026240 |
| } |
| ], |
| "md5sum": "56ed194268a7708733b0cd06c4da19b7" |
| }, |
| { |
| "dataPath": "params_shard_42.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.13.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "3908c105ab14b5ca5619d2a24327a377" |
| }, |
| { |
| "dataPath": "params_shard_43.bin", |
| "format": "raw-shard", |
| "nbytes": 33239040, |
| "records": [ |
| { |
| "name": "transformer.h.12.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.12.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.13.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 15925248 |
| }, |
| { |
| "name": "transformer.h.13.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 15931392 |
| }, |
| { |
| "name": "transformer.h.13.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 28514304 |
| }, |
| { |
| "name": "transformer.h.13.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 30087168 |
| }, |
| { |
| "name": "transformer.h.13.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 33232896 |
| } |
| ], |
| "md5sum": "33f9e20b157bb42d34f121d558c0da70" |
| }, |
| { |
| "dataPath": "params_shard_44.bin", |
| "format": "raw-shard", |
| "nbytes": 21239808, |
| "records": [ |
| { |
| "name": "transformer.h.13.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.13.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 4718592 |
| }, |
| { |
| "name": "transformer.h.13.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 5308416 |
| }, |
| { |
| "name": "transformer.h.13.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 19464192 |
| }, |
| { |
| "name": "transformer.h.14.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 21233664 |
| } |
| ], |
| "md5sum": "1604fa4cc37f0179f444573ede7a5deb" |
| }, |
| { |
| "dataPath": "params_shard_45.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.14.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "2330e02f28e6f4061b18954cb306e561" |
| }, |
| { |
| "dataPath": "params_shard_46.bin", |
| "format": "raw-shard", |
| "nbytes": 22616064, |
| "records": [ |
| { |
| "name": "transformer.h.14.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.14.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 12582912 |
| }, |
| { |
| "name": "transformer.h.14.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.14.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 17301504 |
| }, |
| { |
| "name": "transformer.h.14.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 17307648 |
| }, |
| { |
| "name": "transformer.h.14.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 22026240 |
| } |
| ], |
| "md5sum": "36eeaa4749d1889cd9e7b7221444d03f" |
| }, |
| { |
| "dataPath": "params_shard_47.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.15.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "55cae218fc755e702bdda6f5a540fe35" |
| }, |
| { |
| "dataPath": "params_shard_48.bin", |
| "format": "raw-shard", |
| "nbytes": 33239040, |
| "records": [ |
| { |
| "name": "transformer.h.14.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.14.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.15.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 15925248 |
| }, |
| { |
| "name": "transformer.h.15.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 15931392 |
| }, |
| { |
| "name": "transformer.h.15.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 28514304 |
| }, |
| { |
| "name": "transformer.h.15.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 30087168 |
| }, |
| { |
| "name": "transformer.h.15.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 33232896 |
| } |
| ], |
| "md5sum": "995b6c434af0bf20c41257ac044dcb17" |
| }, |
| { |
| "dataPath": "params_shard_49.bin", |
| "format": "raw-shard", |
| "nbytes": 21239808, |
| "records": [ |
| { |
| "name": "transformer.h.15.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.15.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 4718592 |
| }, |
| { |
| "name": "transformer.h.15.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 5308416 |
| }, |
| { |
| "name": "transformer.h.15.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 19464192 |
| }, |
| { |
| "name": "transformer.h.16.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 21233664 |
| } |
| ], |
| "md5sum": "4cf7d7f6fcff1cde0c15747625a930f2" |
| }, |
| { |
| "dataPath": "params_shard_50.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.16.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "88768cd8dbf6131dd218b737b0fe319e" |
| }, |
| { |
| "dataPath": "params_shard_51.bin", |
| "format": "raw-shard", |
| "nbytes": 22616064, |
| "records": [ |
| { |
| "name": "transformer.h.16.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.16.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 12582912 |
| }, |
| { |
| "name": "transformer.h.16.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.16.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 17301504 |
| }, |
| { |
| "name": "transformer.h.16.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 17307648 |
| }, |
| { |
| "name": "transformer.h.16.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 22026240 |
| } |
| ], |
| "md5sum": "1f06c7b48b49e70fc276c1cebedb7bbb" |
| }, |
| { |
| "dataPath": "params_shard_52.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.17.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "917019b1ae81f19b9ff17e7be8b1c46d" |
| }, |
| { |
| "dataPath": "params_shard_53.bin", |
| "format": "raw-shard", |
| "nbytes": 33239040, |
| "records": [ |
| { |
| "name": "transformer.h.16.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.16.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.17.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 15925248 |
| }, |
| { |
| "name": "transformer.h.17.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 15931392 |
| }, |
| { |
| "name": "transformer.h.17.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 28514304 |
| }, |
| { |
| "name": "transformer.h.17.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 30087168 |
| }, |
| { |
| "name": "transformer.h.17.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 33232896 |
| } |
| ], |
| "md5sum": "49d9fa3f2e97f0c5bacfc79b56dfbf76" |
| }, |
| { |
| "dataPath": "params_shard_54.bin", |
| "format": "raw-shard", |
| "nbytes": 21239808, |
| "records": [ |
| { |
| "name": "transformer.h.17.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.17.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 4718592 |
| }, |
| { |
| "name": "transformer.h.17.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 5308416 |
| }, |
| { |
| "name": "transformer.h.17.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 19464192 |
| }, |
| { |
| "name": "transformer.h.18.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 21233664 |
| } |
| ], |
| "md5sum": "8d38883da369cd5ee82ccfa288fedbf5" |
| }, |
| { |
| "dataPath": "params_shard_55.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.18.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "f07b0ba7133454ef1ff215d1804b42ee" |
| }, |
| { |
| "dataPath": "params_shard_56.bin", |
| "format": "raw-shard", |
| "nbytes": 22616064, |
| "records": [ |
| { |
| "name": "transformer.h.18.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.18.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 12582912 |
| }, |
| { |
| "name": "transformer.h.18.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.18.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 17301504 |
| }, |
| { |
| "name": "transformer.h.18.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 17307648 |
| }, |
| { |
| "name": "transformer.h.18.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 22026240 |
| } |
| ], |
| "md5sum": "8129b573c2878e7021317390fe09409c" |
| }, |
| { |
| "dataPath": "params_shard_57.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.19.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "2d72f59f7c995f4bf601ad7987b9f241" |
| }, |
| { |
| "dataPath": "params_shard_58.bin", |
| "format": "raw-shard", |
| "nbytes": 33239040, |
| "records": [ |
| { |
| "name": "transformer.h.18.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.18.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.19.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 15925248 |
| }, |
| { |
| "name": "transformer.h.19.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 15931392 |
| }, |
| { |
| "name": "transformer.h.19.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 28514304 |
| }, |
| { |
| "name": "transformer.h.19.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 30087168 |
| }, |
| { |
| "name": "transformer.h.19.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 33232896 |
| } |
| ], |
| "md5sum": "32feb4ac2cb6bdf5adb444079e3a3239" |
| }, |
| { |
| "dataPath": "params_shard_59.bin", |
| "format": "raw-shard", |
| "nbytes": 21239808, |
| "records": [ |
| { |
| "name": "transformer.h.19.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.19.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 4718592 |
| }, |
| { |
| "name": "transformer.h.19.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 5308416 |
| }, |
| { |
| "name": "transformer.h.19.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 19464192 |
| }, |
| { |
| "name": "transformer.h.2.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 21233664 |
| } |
| ], |
| "md5sum": "3604c1b91f4cedf6b7f7e47bfdc843e4" |
| }, |
| { |
| "dataPath": "params_shard_60.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.2.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "98300e11ca6111c9098dcc9de6154007" |
| }, |
| { |
| "dataPath": "params_shard_61.bin", |
| "format": "raw-shard", |
| "nbytes": 22616064, |
| "records": [ |
| { |
| "name": "transformer.h.2.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.2.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 12582912 |
| }, |
| { |
| "name": "transformer.h.2.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.2.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 17301504 |
| }, |
| { |
| "name": "transformer.h.2.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 17307648 |
| }, |
| { |
| "name": "transformer.h.2.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 22026240 |
| } |
| ], |
| "md5sum": "b05fbf63ef5ad31f90fb880d8c198a51" |
| }, |
| { |
| "dataPath": "params_shard_62.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.20.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "9ba4f8920d7bf59afdc3c51ad021ed92" |
| }, |
| { |
| "dataPath": "params_shard_63.bin", |
| "format": "raw-shard", |
| "nbytes": 33239040, |
| "records": [ |
| { |
| "name": "transformer.h.2.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.2.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.20.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 15925248 |
| }, |
| { |
| "name": "transformer.h.20.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 15931392 |
| }, |
| { |
| "name": "transformer.h.20.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 28514304 |
| }, |
| { |
| "name": "transformer.h.20.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 30087168 |
| }, |
| { |
| "name": "transformer.h.20.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 33232896 |
| } |
| ], |
| "md5sum": "c4357fc8b560d6779c483631145fa80c" |
| }, |
| { |
| "dataPath": "params_shard_64.bin", |
| "format": "raw-shard", |
| "nbytes": 26548224, |
| "records": [ |
| { |
| "name": "transformer.h.20.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.20.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 4718592 |
| }, |
| { |
| "name": "transformer.h.20.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 5308416 |
| }, |
| { |
| "name": "transformer.h.20.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 19464192 |
| }, |
| { |
| "name": "transformer.h.21.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 21233664 |
| }, |
| { |
| "name": "transformer.h.21.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 25952256 |
| }, |
| { |
| "name": "transformer.h.3.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 26542080 |
| } |
| ], |
| "md5sum": "36e4c4de6e4806e7cd2dd577e00d9872" |
| }, |
| { |
| "dataPath": "params_shard_65.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.3.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "cd3b102866adb064501e901e4023606e" |
| }, |
| { |
| "dataPath": "params_shard_66.bin", |
| "format": "raw-shard", |
| "nbytes": 22616064, |
| "records": [ |
| { |
| "name": "transformer.h.3.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.3.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 12582912 |
| }, |
| { |
| "name": "transformer.h.3.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.3.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 17301504 |
| }, |
| { |
| "name": "transformer.h.3.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 17307648 |
| }, |
| { |
| "name": "transformer.h.3.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 22026240 |
| } |
| ], |
| "md5sum": "efc7312ededb44e55293a119bc005140" |
| }, |
| { |
| "dataPath": "params_shard_67.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.4.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "99d2df12ac0f50534380eecf23a3c3de" |
| }, |
| { |
| "dataPath": "params_shard_68.bin", |
| "format": "raw-shard", |
| "nbytes": 33239040, |
| "records": [ |
| { |
| "name": "transformer.h.3.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.3.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.4.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 15925248 |
| }, |
| { |
| "name": "transformer.h.4.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 15931392 |
| }, |
| { |
| "name": "transformer.h.4.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 28514304 |
| }, |
| { |
| "name": "transformer.h.4.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 30087168 |
| }, |
| { |
| "name": "transformer.h.4.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 33232896 |
| } |
| ], |
| "md5sum": "cdd91cf59662893a8e8288ab47b32fa7" |
| }, |
| { |
| "dataPath": "params_shard_69.bin", |
| "format": "raw-shard", |
| "nbytes": 21239808, |
| "records": [ |
| { |
| "name": "transformer.h.4.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.4.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 4718592 |
| }, |
| { |
| "name": "transformer.h.4.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 5308416 |
| }, |
| { |
| "name": "transformer.h.4.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 19464192 |
| }, |
| { |
| "name": "transformer.h.5.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 21233664 |
| } |
| ], |
| "md5sum": "30c89e0d2ba510b27ad2eecdb69ed329" |
| }, |
| { |
| "dataPath": "params_shard_70.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.5.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "f8f17a39b5b58070a21f5dae32d5b353" |
| }, |
| { |
| "dataPath": "params_shard_71.bin", |
| "format": "raw-shard", |
| "nbytes": 22616064, |
| "records": [ |
| { |
| "name": "transformer.h.5.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.5.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 12582912 |
| }, |
| { |
| "name": "transformer.h.5.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.5.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 17301504 |
| }, |
| { |
| "name": "transformer.h.5.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 17307648 |
| }, |
| { |
| "name": "transformer.h.5.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 22026240 |
| } |
| ], |
| "md5sum": "626b8086883b01a4e267c95a6b87d054" |
| }, |
| { |
| "dataPath": "params_shard_72.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.6.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "7b010c64b1812476c23014b0f22c7c86" |
| }, |
| { |
| "dataPath": "params_shard_73.bin", |
| "format": "raw-shard", |
| "nbytes": 33239040, |
| "records": [ |
| { |
| "name": "transformer.h.5.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.5.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.6.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 15925248 |
| }, |
| { |
| "name": "transformer.h.6.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 15931392 |
| }, |
| { |
| "name": "transformer.h.6.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 28514304 |
| }, |
| { |
| "name": "transformer.h.6.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 30087168 |
| }, |
| { |
| "name": "transformer.h.6.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 33232896 |
| } |
| ], |
| "md5sum": "6e2cef6c473363f6fbc77598e1d7bda8" |
| }, |
| { |
| "dataPath": "params_shard_74.bin", |
| "format": "raw-shard", |
| "nbytes": 21239808, |
| "records": [ |
| { |
| "name": "transformer.h.6.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.6.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 4718592 |
| }, |
| { |
| "name": "transformer.h.6.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 5308416 |
| }, |
| { |
| "name": "transformer.h.6.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 19464192 |
| }, |
| { |
| "name": "transformer.h.7.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 21233664 |
| } |
| ], |
| "md5sum": "bf0c0bbde1d74b800e21d6064e50d5bb" |
| }, |
| { |
| "dataPath": "params_shard_75.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.7.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "060ba9a25db7e4dcfa972264cf1448a0" |
| }, |
| { |
| "dataPath": "params_shard_76.bin", |
| "format": "raw-shard", |
| "nbytes": 22616064, |
| "records": [ |
| { |
| "name": "transformer.h.7.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.7.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 12582912 |
| }, |
| { |
| "name": "transformer.h.7.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.7.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 17301504 |
| }, |
| { |
| "name": "transformer.h.7.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 17307648 |
| }, |
| { |
| "name": "transformer.h.7.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 22026240 |
| } |
| ], |
| "md5sum": "4eb9955b9b5e4930dd379bdd0737a860" |
| }, |
| { |
| "dataPath": "params_shard_77.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.8.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "65b9831a3c07e0a5d2693ee65fc900d0" |
| }, |
| { |
| "dataPath": "params_shard_78.bin", |
| "format": "raw-shard", |
| "nbytes": 33239040, |
| "records": [ |
| { |
| "name": "transformer.h.7.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.7.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.8.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 15925248 |
| }, |
| { |
| "name": "transformer.h.8.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 15931392 |
| }, |
| { |
| "name": "transformer.h.8.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 28514304 |
| }, |
| { |
| "name": "transformer.h.8.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 30087168 |
| }, |
| { |
| "name": "transformer.h.8.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 33232896 |
| } |
| ], |
| "md5sum": "3b5b808a1edf3ebb45c2e5c000396fb1" |
| }, |
| { |
| "dataPath": "params_shard_79.bin", |
| "format": "raw-shard", |
| "nbytes": 21239808, |
| "records": [ |
| { |
| "name": "transformer.h.8.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.8.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 4718592 |
| }, |
| { |
| "name": "transformer.h.8.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 5308416 |
| }, |
| { |
| "name": "transformer.h.8.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 19464192 |
| }, |
| { |
| "name": "transformer.h.9.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 21233664 |
| } |
| ], |
| "md5sum": "f4acecbd4c88a0f5b6e91e5747b2f573" |
| }, |
| { |
| "dataPath": "params_shard_80.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.9.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "6684c5aadf0f546b1bdec716959167e1" |
| }, |
| { |
| "dataPath": "params_shard_81.bin", |
| "format": "raw-shard", |
| "nbytes": 22616064, |
| "records": [ |
| { |
| "name": "transformer.h.9.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.9.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 12582912 |
| }, |
| { |
| "name": "transformer.h.9.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.9.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 17301504 |
| }, |
| { |
| "name": "transformer.h.9.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 17307648 |
| }, |
| { |
| "name": "transformer.h.9.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 22026240 |
| } |
| ], |
| "md5sum": "ea57b18eac2ba6eb2f776685d5809f71" |
| }, |
| { |
| "dataPath": "params_shard_82.bin", |
| "format": "raw-shard", |
| "nbytes": 15925248, |
| "records": [ |
| { |
| "name": "transformer.h.9.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.9.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| } |
| ], |
| "md5sum": "865b527be9a21333488551e85239379f" |
| } |
| ] |
| } |
