{ "metadata": { "ParamSize": 325, "ParamBytes": 2149644288.0, "BitsPerParam": 4.500600961055312 }, "records": [ { "dataPath": "params_shard_0.bin", "format": "raw-shard", "nbytes": 49250304, "records": [ { "name": "lm_head.q_weight", "shape": [ 32064, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 49250304, "byteOffset": 0 } ], "md5sum": "8b7dc0e29e1c572c3c8c79f3545116a6" }, { "dataPath": "params_shard_1.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.21.mlp.gate_up_proj.q_weight", "shape": [ 16384, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "291fa3d154d803e65ea132a1a148ba94" }, { "dataPath": "params_shard_2.bin", "format": "raw-shard", "nbytes": 23470080, "records": [ { "name": "lm_head.q_scale", "shape": [ 32064, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6156288, "byteOffset": 0 }, { "name": "transformer.h.21.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 6156288 }, { "name": "transformer.h.21.mlp.down_proj.q_weight", "shape": [ 3072, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 6162432 }, { "name": "transformer.h.21.mlp.down_proj.q_scale", "shape": [ 3072, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 18745344 }, { "name": "transformer.h.21.mlp.gate_up_proj.q_scale", "shape": [ 16384, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 20318208 }, { "name": "transformer.h.21.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 23463936 } ], "md5sum": "2232562712a502bb30428199cd9d240f" }, { "dataPath": "params_shard_3.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.22.mlp.gate_up_proj.q_weight", "shape": [ 16384, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "6aa51bc82e13a0a09d2bd79ecf706f56" }, { "dataPath": "params_shard_4.bin", "format": "raw-shard", "nbytes": 33239040, "records": [ { "name": "transformer.h.21.mixer.qkv_proj.q_weight", "shape": [ 9216, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 0 }, { "name": "transformer.h.21.mixer.qkv_proj.q_scale", "shape": [ 9216, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1769472, "byteOffset": 14155776 }, { "name": "transformer.h.22.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 15925248 }, { "name": "transformer.h.22.mlp.down_proj.q_weight", "shape": [ 3072, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 15931392 }, { "name": "transformer.h.22.mlp.down_proj.q_scale", "shape": [ 3072, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 28514304 }, { "name": "transformer.h.22.mlp.gate_up_proj.q_scale", "shape": [ 16384, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 30087168 }, { "name": "transformer.h.22.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 33232896 } ], "md5sum": "80a32541f0f5e6d18f5bc3e728a8a81f" }, { "dataPath": "params_shard_5.bin", "format": "raw-shard", "nbytes": 21239808, "records": [ { "name": "transformer.h.22.mixer.out_proj.q_weight", "shape": [ 3072, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 0 }, { "name": "transformer.h.22.mixer.out_proj.q_scale", "shape": [ 3072, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 4718592 }, { "name": "transformer.h.22.mixer.qkv_proj.q_weight", "shape": [ 9216, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 5308416 }, { "name": "transformer.h.22.mixer.qkv_proj.q_scale", "shape": [ 9216, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1769472, "byteOffset": 19464192 }, { "name": "transformer.h.23.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 21233664 } ], "md5sum": "2485cde6690b1ca0f11482f1a3fce948" }, { "dataPath": "params_shard_6.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.23.mlp.gate_up_proj.q_weight", "shape": [ 16384, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "038753ac7a61c5c8fc1f37e48a163184" }, { "dataPath": "params_shard_7.bin", "format": "raw-shard", "nbytes": 22616064, "records": [ { "name": "transformer.h.23.mlp.down_proj.q_weight", "shape": [ 3072, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 0 }, { "name": "transformer.h.23.mlp.down_proj.q_scale", "shape": [ 3072, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 12582912 }, { "name": "transformer.h.23.mlp.gate_up_proj.q_scale", "shape": [ 16384, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 14155776 }, { "name": "transformer.h.23.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 17301504 }, { "name": "transformer.h.23.mixer.out_proj.q_weight", "shape": [ 3072, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 17307648 }, { "name": "transformer.h.23.mixer.out_proj.q_scale", "shape": [ 3072, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 22026240 } ], "md5sum": "7ca7072dc49d39e5028da311c2da59f6" }, { "dataPath": "params_shard_8.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.24.mlp.gate_up_proj.q_weight", "shape": [ 16384, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "10ec010d75596ab4288c0101cdaeaed3" }, { "dataPath": "params_shard_9.bin", "format": "raw-shard", "nbytes": 33239040, "records": [ { "name": "transformer.h.23.mixer.qkv_proj.q_weight", "shape": [ 9216, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 0 }, { "name": "transformer.h.23.mixer.qkv_proj.q_scale", "shape": [ 9216, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1769472, "byteOffset": 14155776 }, { "name": "transformer.h.24.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 15925248 }, { "name": "transformer.h.24.mlp.down_proj.q_weight", "shape": [ 3072, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 15931392 }, { "name": "transformer.h.24.mlp.down_proj.q_scale", "shape": [ 3072, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 28514304 }, { "name": "transformer.h.24.mlp.gate_up_proj.q_scale", "shape": [ 16384, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 30087168 }, { "name": "transformer.h.24.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 33232896 } ], "md5sum": "f4ab5f8b0b0d589d9b1411243a381d97" }, { "dataPath": "params_shard_10.bin", "format": "raw-shard", "nbytes": 21239808, "records": [ { "name": "transformer.h.24.mixer.out_proj.q_weight", "shape": [ 3072, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 0 }, { "name": "transformer.h.24.mixer.out_proj.q_scale", "shape": [ 3072, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 4718592 }, { "name": "transformer.h.24.mixer.qkv_proj.q_weight", "shape": [ 9216, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 5308416 }, { "name": "transformer.h.24.mixer.qkv_proj.q_scale", "shape": [ 9216, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1769472, "byteOffset": 19464192 }, { "name": "transformer.h.25.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 21233664 } ], "md5sum": "3de4e4bb86177439388236f247893c0c" }, { "dataPath": "params_shard_11.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.25.mlp.gate_up_proj.q_weight", "shape": [ 16384, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "d0e4431ab49bbe7282f00262b88e11ac" }, { "dataPath": "params_shard_12.bin", "format": "raw-shard", "nbytes": 22616064, "records": [ { "name": "transformer.h.25.mlp.down_proj.q_weight", "shape": [ 3072, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 0 }, { "name": "transformer.h.25.mlp.down_proj.q_scale", "shape": [ 3072, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 12582912 }, { "name": "transformer.h.25.mlp.gate_up_proj.q_scale", "shape": [ 16384, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 14155776 }, { "name": "transformer.h.25.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 17301504 }, { "name": "transformer.h.25.mixer.out_proj.q_weight", "shape": [ 3072, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 17307648 }, { "name": "transformer.h.25.mixer.out_proj.q_scale", "shape": [ 3072, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 22026240 } ], "md5sum": "21ffcb98e9f9e5b7eedbb6e92c661716" }, { "dataPath": "params_shard_13.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.26.mlp.gate_up_proj.q_weight", "shape": [ 16384, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "244eba2a84e4a3bc56632585f826328b" }, { "dataPath": "params_shard_14.bin", "format": "raw-shard", "nbytes": 33239040, "records": [ { "name": "transformer.h.25.mixer.qkv_proj.q_weight", "shape": [ 9216, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 0 }, { "name": "transformer.h.25.mixer.qkv_proj.q_scale", "shape": [ 9216, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1769472, "byteOffset": 14155776 }, { "name": "transformer.h.26.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 15925248 }, { "name": "transformer.h.26.mlp.down_proj.q_weight", "shape": [ 3072, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 15931392 }, { "name": "transformer.h.26.mlp.down_proj.q_scale", "shape": [ 3072, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 28514304 }, { "name": "transformer.h.26.mlp.gate_up_proj.q_scale", "shape": [ 16384, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 30087168 }, { "name": "transformer.h.26.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 33232896 } ], "md5sum": "2a9f1879dcbdb4843449b6b588089233" }, { "dataPath": "params_shard_15.bin", "format": "raw-shard", "nbytes": 21239808, "records": [ { "name": "transformer.h.26.mixer.out_proj.q_weight", "shape": [ 3072, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 0 }, { "name": "transformer.h.26.mixer.out_proj.q_scale", "shape": [ 3072, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 4718592 }, { "name": "transformer.h.26.mixer.qkv_proj.q_weight", "shape": [ 9216, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 5308416 }, { "name": "transformer.h.26.mixer.qkv_proj.q_scale", "shape": [ 9216, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1769472, "byteOffset": 19464192 }, { "name": "transformer.h.27.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 21233664 } ], "md5sum": "db94e9d2b995403fc11c3fe2573463f4" }, { "dataPath": "params_shard_16.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.27.mlp.gate_up_proj.q_weight", "shape": [ 16384, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "e77dfa662ddf8497df0f796e0dfb7797" }, { "dataPath": "params_shard_17.bin", "format": "raw-shard", "nbytes": 22616064, "records": [ { "name": "transformer.h.27.mlp.down_proj.q_weight", "shape": [ 3072, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 0 }, { "name": "transformer.h.27.mlp.down_proj.q_scale", "shape": [ 3072, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 12582912 }, { "name": "transformer.h.27.mlp.gate_up_proj.q_scale", "shape": [ 16384, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 14155776 }, { "name": "transformer.h.27.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 17301504 }, { "name": "transformer.h.27.mixer.out_proj.q_weight", "shape": [ 3072, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 17307648 }, { "name": "transformer.h.27.mixer.out_proj.q_scale", "shape": [ 3072, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 22026240 } ], "md5sum": "61062289a20a74ca7acce143db7ff09f" }, { "dataPath": "params_shard_18.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.28.mlp.gate_up_proj.q_weight", "shape": [ 16384, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "468d07c6c0d4decfadcb24eb2384f21f" }, { "dataPath": "params_shard_19.bin", "format": "raw-shard", "nbytes": 33239040, "records": [ { "name": "transformer.h.27.mixer.qkv_proj.q_weight", "shape": [ 9216, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 0 }, { "name": "transformer.h.27.mixer.qkv_proj.q_scale", "shape": [ 9216, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1769472, "byteOffset": 14155776 }, { "name": "transformer.h.28.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 15925248 }, { "name": "transformer.h.28.mlp.down_proj.q_weight", "shape": [ 3072, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 15931392 }, { "name": "transformer.h.28.mlp.down_proj.q_scale", "shape": [ 3072, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 28514304 }, { "name": "transformer.h.28.mlp.gate_up_proj.q_scale", "shape": [ 16384, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 30087168 }, { "name": "transformer.h.28.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 33232896 } ], "md5sum": "7ea9ba97de223dec60d8d652a4de4f93" }, { "dataPath": "params_shard_20.bin", "format": "raw-shard", "nbytes": 21239808, "records": [ { "name": "transformer.h.28.mixer.out_proj.q_weight", "shape": [ 3072, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 0 }, { "name": "transformer.h.28.mixer.out_proj.q_scale", "shape": [ 3072, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 4718592 }, { "name": "transformer.h.28.mixer.qkv_proj.q_weight", "shape": [ 9216, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 5308416 }, { "name": "transformer.h.28.mixer.qkv_proj.q_scale", "shape": [ 9216, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1769472, "byteOffset": 19464192 }, { "name": "transformer.h.29.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 21233664 } ], "md5sum": "7309354a4d39fe28c714851b250b7f13" }, { "dataPath": "params_shard_21.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.29.mlp.gate_up_proj.q_weight", "shape": [ 16384, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "e516301a253bb0a02a4cfabc742f095a" }, { "dataPath": "params_shard_22.bin", "format": "raw-shard", "nbytes": 22616064, "records": [ { "name": "transformer.h.29.mlp.down_proj.q_weight", "shape": [ 3072, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 0 }, { "name": "transformer.h.29.mlp.down_proj.q_scale", "shape": [ 3072, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 12582912 }, { "name": "transformer.h.29.mlp.gate_up_proj.q_scale", "shape": [ 16384, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 14155776 }, { "name": "transformer.h.29.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 17301504 }, { "name": "transformer.h.29.mixer.out_proj.q_weight", "shape": [ 3072, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 17307648 }, { "name": "transformer.h.29.mixer.out_proj.q_scale", "shape": [ 3072, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 22026240 } ], "md5sum": "519ceaa2aa175d7cea5af5d4d22894bc" }, { "dataPath": "params_shard_23.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.30.mlp.gate_up_proj.q_weight", "shape": [ 16384, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "b7a3d71b718be0cd079fc4592ae36cee" }, { "dataPath": "params_shard_24.bin", "format": "raw-shard", "nbytes": 33239040, "records": [ { "name": "transformer.h.29.mixer.qkv_proj.q_weight", "shape": [ 9216, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 0 }, { "name": "transformer.h.29.mixer.qkv_proj.q_scale", "shape": [ 9216, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1769472, "byteOffset": 14155776 }, { "name": "transformer.h.30.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 15925248 }, { "name": "transformer.h.30.mlp.down_proj.q_weight", "shape": [ 3072, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 15931392 }, { "name": "transformer.h.30.mlp.down_proj.q_scale", "shape": [ 3072, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 28514304 }, { "name": "transformer.h.30.mlp.gate_up_proj.q_scale", "shape": [ 16384, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 30087168 }, { "name": "transformer.h.30.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 33232896 } ], "md5sum": "51694630cfe5563d88f7129d486080e1" }, { "dataPath": "params_shard_25.bin", "format": "raw-shard", "nbytes": 21239808, "records": [ { "name": "transformer.h.30.mixer.out_proj.q_weight", "shape": [ 3072, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 0 }, { "name": "transformer.h.30.mixer.out_proj.q_scale", "shape": [ 3072, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 4718592 }, { "name": "transformer.h.30.mixer.qkv_proj.q_weight", "shape": [ 9216, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 5308416 }, { "name": "transformer.h.30.mixer.qkv_proj.q_scale", "shape": [ 9216, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1769472, "byteOffset": 19464192 }, { "name": "transformer.h.31.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 21233664 } ], "md5sum": "52c95231effd3abcc0010bd99f64b661" }, { "dataPath": "params_shard_26.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.31.mlp.gate_up_proj.q_weight", "shape": [ 16384, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "42aec8a789db093d5a850365beca1b9a" }, { "dataPath": "params_shard_27.bin", "format": "raw-shard", "nbytes": 22616064, "records": [ { "name": "transformer.h.31.mlp.down_proj.q_weight", "shape": [ 3072, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 0 }, { "name": "transformer.h.31.mlp.down_proj.q_scale", "shape": [ 3072, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 12582912 }, { "name": "transformer.h.31.mlp.gate_up_proj.q_scale", "shape": [ 16384, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 14155776 }, { "name": "transformer.h.31.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 17301504 }, { "name": "transformer.h.31.mixer.out_proj.q_weight", "shape": [ 3072, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 17307648 }, { "name": "transformer.h.31.mixer.out_proj.q_scale", "shape": [ 3072, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 22026240 } ], "md5sum": "d0a102a92f5c015bb1e71cdeb632fe0c" }, { "dataPath": "params_shard_28.bin", "format": "raw-shard", "nbytes": 49250304, "records": [ { "name": "transformer.embd.q_weight", "shape": [ 32064, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 49250304, "byteOffset": 0 } ], "md5sum": "e65cd6e13f78175ce2f5fcb3844136e4" }, { "dataPath": "params_shard_29.bin", "format": "raw-shard", "nbytes": 22093824, "records": [ { "name": "transformer.h.31.mixer.qkv_proj.q_weight", "shape": [ 9216, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 0 }, { "name": "transformer.h.31.mixer.qkv_proj.q_scale", "shape": [ 9216, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1769472, "byteOffset": 14155776 }, { "name": "transformer.norm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 15925248 }, { "name": "transformer.embd.q_scale", "shape": [ 32064, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6156288, "byteOffset": 15931392 }, { "name": "transformer.h.0.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 22087680 } ], "md5sum": "13906ed5091c3b4cabe7f7448ea53d40" }, { "dataPath": "params_shard_30.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.0.mlp.gate_up_proj.q_weight", "shape": [ 16384, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "01a5d0a50e64f55fba559fee50cf9fa9" }, { "dataPath": "params_shard_31.bin", "format": "raw-shard", "nbytes": 22616064, "records": [ { "name": "transformer.h.0.mlp.down_proj.q_weight", "shape": [ 3072, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 0 }, { "name": "transformer.h.0.mlp.down_proj.q_scale", "shape": [ 3072, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 12582912 }, { "name": "transformer.h.0.mlp.gate_up_proj.q_scale", "shape": [ 16384, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 14155776 }, { "name": "transformer.h.0.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 17301504 }, { "name": "transformer.h.0.mixer.out_proj.q_weight", "shape": [ 3072, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 17307648 }, { "name": "transformer.h.0.mixer.out_proj.q_scale", "shape": [ 3072, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 22026240 } ], "md5sum": "629271dbb7fcacc486a2b5191b5457c7" }, { "dataPath": "params_shard_32.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.1.mlp.gate_up_proj.q_weight", "shape": [ 16384, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "06b55aec3bd00e550e033e2c8d6ac0ac" }, { "dataPath": "params_shard_33.bin", "format": "raw-shard", "nbytes": 33239040, "records": [ { "name": "transformer.h.0.mixer.qkv_proj.q_weight", "shape": [ 9216, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 0 }, { "name": "transformer.h.0.mixer.qkv_proj.q_scale", "shape": [ 9216, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1769472, "byteOffset": 14155776 }, { "name": "transformer.h.1.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 15925248 }, { "name": "transformer.h.1.mlp.down_proj.q_weight", "shape": [ 3072, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 15931392 }, { "name": "transformer.h.1.mlp.down_proj.q_scale", "shape": [ 3072, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 28514304 }, { "name": "transformer.h.1.mlp.gate_up_proj.q_scale", "shape": [ 16384, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 30087168 }, { "name": "transformer.h.1.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 33232896 } ], "md5sum": "0baa80591c549d23d1a9176e7c8d8672" }, { "dataPath": "params_shard_34.bin", "format": "raw-shard", "nbytes": 21239808, "records": [ { "name": "transformer.h.1.mixer.out_proj.q_weight", "shape": [ 3072, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 0 }, { "name": "transformer.h.1.mixer.out_proj.q_scale", "shape": [ 3072, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 4718592 }, { "name": "transformer.h.1.mixer.qkv_proj.q_weight", "shape": [ 9216, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 5308416 }, { "name": "transformer.h.1.mixer.qkv_proj.q_scale", "shape": [ 9216, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1769472, "byteOffset": 19464192 }, { "name": "transformer.h.10.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 21233664 } ], "md5sum": "5584b898584835fcd5480db5f0e4975f" }, { "dataPath": "params_shard_35.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.10.mlp.gate_up_proj.q_weight", "shape": [ 16384, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "34d6d5bb2604bf2c325306582370cc74" }, { "dataPath": "params_shard_36.bin", "format": "raw-shard", "nbytes": 22616064, "records": [ { "name": "transformer.h.10.mlp.down_proj.q_weight", "shape": [ 3072, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 0 }, { "name": "transformer.h.10.mlp.down_proj.q_scale", "shape": [ 3072, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 12582912 }, { "name": "transformer.h.10.mlp.gate_up_proj.q_scale", "shape": [ 16384, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 14155776 }, { "name": "transformer.h.10.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 17301504 }, { "name": "transformer.h.10.mixer.out_proj.q_weight", "shape": [ 3072, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 17307648 }, { "name": "transformer.h.10.mixer.out_proj.q_scale", "shape": [ 3072, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 22026240 } ], "md5sum": "0dda9f13d0d93ef2639737ef153eba07" }, { "dataPath": "params_shard_37.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.11.mlp.gate_up_proj.q_weight", "shape": [ 16384, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "978d47b4e8ab28da90916bdf8b5fd352" }, { "dataPath": "params_shard_38.bin", "format": "raw-shard", "nbytes": 33239040, "records": [ { "name": "transformer.h.10.mixer.qkv_proj.q_weight", "shape": [ 9216, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 0 }, { "name": "transformer.h.10.mixer.qkv_proj.q_scale", "shape": [ 9216, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1769472, "byteOffset": 14155776 }, { "name": "transformer.h.11.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 15925248 }, { "name": "transformer.h.11.mlp.down_proj.q_weight", "shape": [ 3072, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 15931392 }, { "name": "transformer.h.11.mlp.down_proj.q_scale", "shape": [ 3072, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 28514304 }, { "name": "transformer.h.11.mlp.gate_up_proj.q_scale", "shape": [ 16384, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 30087168 }, { "name": "transformer.h.11.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 33232896 } ], "md5sum": "44b1c9c79d2b4eee0bc34d275fe175fa" }, { "dataPath": "params_shard_39.bin", "format": "raw-shard", "nbytes": 21239808, "records": [ { "name": "transformer.h.11.mixer.out_proj.q_weight", "shape": [ 3072, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 0 }, { "name": "transformer.h.11.mixer.out_proj.q_scale", "shape": [ 3072, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 4718592 }, { "name": "transformer.h.11.mixer.qkv_proj.q_weight", "shape": [ 9216, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 5308416 }, { "name": "transformer.h.11.mixer.qkv_proj.q_scale", "shape": [ 9216, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1769472, "byteOffset": 19464192 }, { "name": "transformer.h.12.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 21233664 } ], "md5sum": "4666acf541e4ffd237a29411256a97d2" }, { "dataPath": "params_shard_40.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.12.mlp.gate_up_proj.q_weight", "shape": [ 16384, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "e79ed66deccd5e690ccfe389f77031a2" }, { "dataPath": "params_shard_41.bin", "format": "raw-shard", "nbytes": 22616064, "records": [ { "name": "transformer.h.12.mlp.down_proj.q_weight", "shape": [ 3072, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 0 }, { "name": "transformer.h.12.mlp.down_proj.q_scale", "shape": [ 3072, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 12582912 }, { "name": "transformer.h.12.mlp.gate_up_proj.q_scale", "shape": [ 16384, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 14155776 }, { "name": "transformer.h.12.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 17301504 }, { "name": "transformer.h.12.mixer.out_proj.q_weight", "shape": [ 3072, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 17307648 }, { "name": "transformer.h.12.mixer.out_proj.q_scale", "shape": [ 3072, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 22026240 } ], "md5sum": "56ed194268a7708733b0cd06c4da19b7" }, { "dataPath": "params_shard_42.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.13.mlp.gate_up_proj.q_weight", "shape": [ 16384, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "3908c105ab14b5ca5619d2a24327a377" }, { "dataPath": "params_shard_43.bin", "format": "raw-shard", "nbytes": 33239040, "records": [ { "name": "transformer.h.12.mixer.qkv_proj.q_weight", "shape": [ 9216, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 0 }, { "name": "transformer.h.12.mixer.qkv_proj.q_scale", "shape": [ 9216, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1769472, "byteOffset": 14155776 }, { "name": "transformer.h.13.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 15925248 }, { "name": "transformer.h.13.mlp.down_proj.q_weight", "shape": [ 3072, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 15931392 }, { "name": "transformer.h.13.mlp.down_proj.q_scale", "shape": [ 3072, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 28514304 }, { "name": "transformer.h.13.mlp.gate_up_proj.q_scale", "shape": [ 16384, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 30087168 }, { "name": "transformer.h.13.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 33232896 } ], "md5sum": "33f9e20b157bb42d34f121d558c0da70" }, { "dataPath": "params_shard_44.bin", "format": "raw-shard", "nbytes": 21239808, "records": [ { "name": "transformer.h.13.mixer.out_proj.q_weight", "shape": [ 3072, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 0 }, { "name": "transformer.h.13.mixer.out_proj.q_scale", "shape": [ 3072, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 4718592 }, { "name": "transformer.h.13.mixer.qkv_proj.q_weight", "shape": [ 9216, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 5308416 }, { "name": "transformer.h.13.mixer.qkv_proj.q_scale", "shape": [ 9216, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1769472, "byteOffset": 19464192 }, { "name": "transformer.h.14.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 21233664 } ], "md5sum": "1604fa4cc37f0179f444573ede7a5deb" }, { "dataPath": "params_shard_45.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.14.mlp.gate_up_proj.q_weight", "shape": [ 16384, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "2330e02f28e6f4061b18954cb306e561" }, { "dataPath": "params_shard_46.bin", "format": "raw-shard", "nbytes": 22616064, "records": [ { "name": "transformer.h.14.mlp.down_proj.q_weight", "shape": [ 3072, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 0 }, { "name": "transformer.h.14.mlp.down_proj.q_scale", "shape": [ 3072, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 12582912 }, { "name": "transformer.h.14.mlp.gate_up_proj.q_scale", "shape": [ 16384, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 14155776 }, { "name": "transformer.h.14.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 17301504 }, { "name": "transformer.h.14.mixer.out_proj.q_weight", "shape": [ 3072, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 17307648 }, { "name": "transformer.h.14.mixer.out_proj.q_scale", "shape": [ 3072, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 22026240 } ], "md5sum": "36eeaa4749d1889cd9e7b7221444d03f" }, { "dataPath": "params_shard_47.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.15.mlp.gate_up_proj.q_weight", "shape": [ 16384, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "55cae218fc755e702bdda6f5a540fe35" }, { "dataPath": "params_shard_48.bin", "format": "raw-shard", "nbytes": 33239040, "records": [ { "name": "transformer.h.14.mixer.qkv_proj.q_weight", "shape": [ 9216, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 0 }, { "name": "transformer.h.14.mixer.qkv_proj.q_scale", "shape": [ 9216, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1769472, "byteOffset": 14155776 }, { "name": "transformer.h.15.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 15925248 }, { "name": "transformer.h.15.mlp.down_proj.q_weight", "shape": [ 3072, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 15931392 }, { "name": "transformer.h.15.mlp.down_proj.q_scale", "shape": [ 3072, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 28514304 }, { "name": "transformer.h.15.mlp.gate_up_proj.q_scale", "shape": [ 16384, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 30087168 }, { "name": "transformer.h.15.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 33232896 } ], "md5sum": "995b6c434af0bf20c41257ac044dcb17" }, { "dataPath": "params_shard_49.bin", "format": "raw-shard", "nbytes": 21239808, "records": [ { "name": "transformer.h.15.mixer.out_proj.q_weight", "shape": [ 3072, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 0 }, { "name": "transformer.h.15.mixer.out_proj.q_scale", "shape": [ 3072, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 4718592 }, { "name": "transformer.h.15.mixer.qkv_proj.q_weight", "shape": [ 9216, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 5308416 }, { "name": "transformer.h.15.mixer.qkv_proj.q_scale", "shape": [ 9216, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1769472, "byteOffset": 19464192 }, { "name": "transformer.h.16.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 21233664 } ], "md5sum": "4cf7d7f6fcff1cde0c15747625a930f2" }, { "dataPath": "params_shard_50.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.16.mlp.gate_up_proj.q_weight", "shape": [ 16384, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "88768cd8dbf6131dd218b737b0fe319e" }, { "dataPath": "params_shard_51.bin", "format": "raw-shard", "nbytes": 22616064, "records": [ { "name": "transformer.h.16.mlp.down_proj.q_weight", "shape": [ 3072, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 0 }, { "name": "transformer.h.16.mlp.down_proj.q_scale", "shape": [ 3072, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 12582912 }, { "name": "transformer.h.16.mlp.gate_up_proj.q_scale", "shape": [ 16384, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 14155776 }, { "name": "transformer.h.16.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 17301504 }, { "name": "transformer.h.16.mixer.out_proj.q_weight", "shape": [ 3072, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 17307648 }, { "name": "transformer.h.16.mixer.out_proj.q_scale", "shape": [ 3072, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 22026240 } ], "md5sum": "1f06c7b48b49e70fc276c1cebedb7bbb" }, { "dataPath": "params_shard_52.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.17.mlp.gate_up_proj.q_weight", "shape": [ 16384, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "917019b1ae81f19b9ff17e7be8b1c46d" }, { "dataPath": "params_shard_53.bin", "format": "raw-shard", "nbytes": 33239040, "records": [ { "name": "transformer.h.16.mixer.qkv_proj.q_weight", "shape": [ 9216, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 0 }, { "name": "transformer.h.16.mixer.qkv_proj.q_scale", "shape": [ 9216, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1769472, "byteOffset": 14155776 }, { "name": "transformer.h.17.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 15925248 }, { "name": "transformer.h.17.mlp.down_proj.q_weight", "shape": [ 3072, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 15931392 }, { "name": "transformer.h.17.mlp.down_proj.q_scale", "shape": [ 3072, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 28514304 }, { "name": "transformer.h.17.mlp.gate_up_proj.q_scale", "shape": [ 16384, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 30087168 }, { "name": "transformer.h.17.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 33232896 } ], "md5sum": "49d9fa3f2e97f0c5bacfc79b56dfbf76" }, { "dataPath": "params_shard_54.bin", "format": "raw-shard", "nbytes": 21239808, "records": [ { "name": "transformer.h.17.mixer.out_proj.q_weight", "shape": [ 3072, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 0 }, { "name": "transformer.h.17.mixer.out_proj.q_scale", "shape": [ 3072, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 4718592 }, { "name": "transformer.h.17.mixer.qkv_proj.q_weight", "shape": [ 9216, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 5308416 }, { "name": "transformer.h.17.mixer.qkv_proj.q_scale", "shape": [ 9216, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1769472, "byteOffset": 19464192 }, { "name": "transformer.h.18.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 21233664 } ], "md5sum": "8d38883da369cd5ee82ccfa288fedbf5" }, { "dataPath": "params_shard_55.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.18.mlp.gate_up_proj.q_weight", "shape": [ 16384, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "f07b0ba7133454ef1ff215d1804b42ee" }, { "dataPath": "params_shard_56.bin", "format": "raw-shard", "nbytes": 22616064, "records": [ { "name": "transformer.h.18.mlp.down_proj.q_weight", "shape": [ 3072, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 0 }, { "name": "transformer.h.18.mlp.down_proj.q_scale", "shape": [ 3072, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 12582912 }, { "name": "transformer.h.18.mlp.gate_up_proj.q_scale", "shape": [ 16384, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 14155776 }, { "name": "transformer.h.18.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 17301504 }, { "name": "transformer.h.18.mixer.out_proj.q_weight", "shape": [ 3072, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 17307648 }, { "name": "transformer.h.18.mixer.out_proj.q_scale", "shape": [ 3072, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 22026240 } ], "md5sum": "8129b573c2878e7021317390fe09409c" }, { "dataPath": "params_shard_57.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.19.mlp.gate_up_proj.q_weight", "shape": [ 16384, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "2d72f59f7c995f4bf601ad7987b9f241" }, { "dataPath": "params_shard_58.bin", "format": "raw-shard", "nbytes": 33239040, "records": [ { "name": "transformer.h.18.mixer.qkv_proj.q_weight", "shape": [ 9216, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 0 }, { "name": "transformer.h.18.mixer.qkv_proj.q_scale", "shape": [ 9216, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1769472, "byteOffset": 14155776 }, { "name": "transformer.h.19.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 15925248 }, { "name": "transformer.h.19.mlp.down_proj.q_weight", "shape": [ 3072, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 15931392 }, { "name": "transformer.h.19.mlp.down_proj.q_scale", "shape": [ 3072, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 28514304 }, { "name": "transformer.h.19.mlp.gate_up_proj.q_scale", "shape": [ 16384, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 30087168 }, { "name": "transformer.h.19.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 33232896 } ], "md5sum": "32feb4ac2cb6bdf5adb444079e3a3239" }, { "dataPath": "params_shard_59.bin", "format": "raw-shard", "nbytes": 21239808, "records": [ { "name": "transformer.h.19.mixer.out_proj.q_weight", "shape": [ 3072, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 0 }, { "name": "transformer.h.19.mixer.out_proj.q_scale", "shape": [ 3072, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 4718592 }, { "name": "transformer.h.19.mixer.qkv_proj.q_weight", "shape": [ 9216, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 5308416 }, { "name": "transformer.h.19.mixer.qkv_proj.q_scale", "shape": [ 9216, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1769472, "byteOffset": 19464192 }, { "name": "transformer.h.2.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 21233664 } ], "md5sum": "3604c1b91f4cedf6b7f7e47bfdc843e4" }, { "dataPath": "params_shard_60.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.2.mlp.gate_up_proj.q_weight", "shape": [ 16384, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "98300e11ca6111c9098dcc9de6154007" }, { "dataPath": "params_shard_61.bin", "format": "raw-shard", "nbytes": 22616064, "records": [ { "name": "transformer.h.2.mlp.down_proj.q_weight", "shape": [ 3072, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 0 }, { "name": "transformer.h.2.mlp.down_proj.q_scale", "shape": [ 3072, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 12582912 }, { "name": "transformer.h.2.mlp.gate_up_proj.q_scale", "shape": [ 16384, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 14155776 }, { "name": "transformer.h.2.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 17301504 }, { "name": "transformer.h.2.mixer.out_proj.q_weight", "shape": [ 3072, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 17307648 }, { "name": "transformer.h.2.mixer.out_proj.q_scale", "shape": [ 3072, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 22026240 } ], "md5sum": "b05fbf63ef5ad31f90fb880d8c198a51" }, { "dataPath": "params_shard_62.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.20.mlp.gate_up_proj.q_weight", "shape": [ 16384, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "9ba4f8920d7bf59afdc3c51ad021ed92" }, { "dataPath": "params_shard_63.bin", "format": "raw-shard", "nbytes": 33239040, "records": [ { "name": "transformer.h.2.mixer.qkv_proj.q_weight", "shape": [ 9216, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 0 }, { "name": "transformer.h.2.mixer.qkv_proj.q_scale", "shape": [ 9216, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1769472, "byteOffset": 14155776 }, { "name": "transformer.h.20.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 15925248 }, { "name": "transformer.h.20.mlp.down_proj.q_weight", "shape": [ 3072, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 15931392 }, { "name": "transformer.h.20.mlp.down_proj.q_scale", "shape": [ 3072, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 28514304 }, { "name": "transformer.h.20.mlp.gate_up_proj.q_scale", "shape": [ 16384, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 30087168 }, { "name": "transformer.h.20.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 33232896 } ], "md5sum": "c4357fc8b560d6779c483631145fa80c" }, { "dataPath": "params_shard_64.bin", "format": "raw-shard", "nbytes": 26548224, "records": [ { "name": "transformer.h.20.mixer.out_proj.q_weight", "shape": [ 3072, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 0 }, { "name": "transformer.h.20.mixer.out_proj.q_scale", "shape": [ 3072, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 4718592 }, { "name": "transformer.h.20.mixer.qkv_proj.q_weight", "shape": [ 9216, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 5308416 }, { "name": "transformer.h.20.mixer.qkv_proj.q_scale", "shape": [ 9216, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1769472, "byteOffset": 19464192 }, { "name": "transformer.h.21.mixer.out_proj.q_weight", "shape": [ 3072, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 21233664 }, { "name": "transformer.h.21.mixer.out_proj.q_scale", "shape": [ 3072, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 25952256 }, { "name": "transformer.h.3.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 26542080 } ], "md5sum": "36e4c4de6e4806e7cd2dd577e00d9872" }, { "dataPath": "params_shard_65.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.3.mlp.gate_up_proj.q_weight", "shape": [ 16384, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "cd3b102866adb064501e901e4023606e" }, { "dataPath": "params_shard_66.bin", "format": "raw-shard", "nbytes": 22616064, "records": [ { "name": "transformer.h.3.mlp.down_proj.q_weight", "shape": [ 3072, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 0 }, { "name": "transformer.h.3.mlp.down_proj.q_scale", "shape": [ 3072, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 12582912 }, { "name": "transformer.h.3.mlp.gate_up_proj.q_scale", "shape": [ 16384, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 14155776 }, { "name": "transformer.h.3.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 17301504 }, { "name": "transformer.h.3.mixer.out_proj.q_weight", "shape": [ 3072, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 17307648 }, { "name": "transformer.h.3.mixer.out_proj.q_scale", "shape": [ 3072, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 22026240 } ], "md5sum": "efc7312ededb44e55293a119bc005140" }, { "dataPath": "params_shard_67.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.4.mlp.gate_up_proj.q_weight", "shape": [ 16384, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "99d2df12ac0f50534380eecf23a3c3de" }, { "dataPath": "params_shard_68.bin", "format": "raw-shard", "nbytes": 33239040, "records": [ { "name": "transformer.h.3.mixer.qkv_proj.q_weight", "shape": [ 9216, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 0 }, { "name": "transformer.h.3.mixer.qkv_proj.q_scale", "shape": [ 9216, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1769472, "byteOffset": 14155776 }, { "name": "transformer.h.4.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 15925248 }, { "name": "transformer.h.4.mlp.down_proj.q_weight", "shape": [ 3072, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 15931392 }, { "name": "transformer.h.4.mlp.down_proj.q_scale", "shape": [ 3072, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 28514304 }, { "name": "transformer.h.4.mlp.gate_up_proj.q_scale", "shape": [ 16384, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 30087168 }, { "name": "transformer.h.4.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 33232896 } ], "md5sum": "cdd91cf59662893a8e8288ab47b32fa7" }, { "dataPath": "params_shard_69.bin", "format": "raw-shard", "nbytes": 21239808, "records": [ { "name": "transformer.h.4.mixer.out_proj.q_weight", "shape": [ 3072, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 0 }, { "name": "transformer.h.4.mixer.out_proj.q_scale", "shape": [ 3072, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 4718592 }, { "name": "transformer.h.4.mixer.qkv_proj.q_weight", "shape": [ 9216, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 5308416 }, { "name": "transformer.h.4.mixer.qkv_proj.q_scale", "shape": [ 9216, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1769472, "byteOffset": 19464192 }, { "name": "transformer.h.5.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 21233664 } ], "md5sum": "30c89e0d2ba510b27ad2eecdb69ed329" }, { "dataPath": "params_shard_70.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.5.mlp.gate_up_proj.q_weight", "shape": [ 16384, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "f8f17a39b5b58070a21f5dae32d5b353" }, { "dataPath": "params_shard_71.bin", "format": "raw-shard", "nbytes": 22616064, "records": [ { "name": "transformer.h.5.mlp.down_proj.q_weight", "shape": [ 3072, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 0 }, { "name": "transformer.h.5.mlp.down_proj.q_scale", "shape": [ 3072, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 12582912 }, { "name": "transformer.h.5.mlp.gate_up_proj.q_scale", "shape": [ 16384, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 14155776 }, { "name": "transformer.h.5.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 17301504 }, { "name": "transformer.h.5.mixer.out_proj.q_weight", "shape": [ 3072, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 17307648 }, { "name": "transformer.h.5.mixer.out_proj.q_scale", "shape": [ 3072, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 22026240 } ], "md5sum": "626b8086883b01a4e267c95a6b87d054" }, { "dataPath": "params_shard_72.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.6.mlp.gate_up_proj.q_weight", "shape": [ 16384, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "7b010c64b1812476c23014b0f22c7c86" }, { "dataPath": "params_shard_73.bin", "format": "raw-shard", "nbytes": 33239040, "records": [ { "name": "transformer.h.5.mixer.qkv_proj.q_weight", "shape": [ 9216, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 0 }, { "name": "transformer.h.5.mixer.qkv_proj.q_scale", "shape": [ 9216, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1769472, "byteOffset": 14155776 }, { "name": "transformer.h.6.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 15925248 }, { "name": "transformer.h.6.mlp.down_proj.q_weight", "shape": [ 3072, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 15931392 }, { "name": "transformer.h.6.mlp.down_proj.q_scale", "shape": [ 3072, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 28514304 }, { "name": "transformer.h.6.mlp.gate_up_proj.q_scale", "shape": [ 16384, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 30087168 }, { "name": "transformer.h.6.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 33232896 } ], "md5sum": "6e2cef6c473363f6fbc77598e1d7bda8" }, { "dataPath": "params_shard_74.bin", "format": "raw-shard", "nbytes": 21239808, "records": [ { "name": "transformer.h.6.mixer.out_proj.q_weight", "shape": [ 3072, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 0 }, { "name": "transformer.h.6.mixer.out_proj.q_scale", "shape": [ 3072, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 4718592 }, { "name": "transformer.h.6.mixer.qkv_proj.q_weight", "shape": [ 9216, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 5308416 }, { "name": "transformer.h.6.mixer.qkv_proj.q_scale", "shape": [ 9216, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1769472, "byteOffset": 19464192 }, { "name": "transformer.h.7.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 21233664 } ], "md5sum": "bf0c0bbde1d74b800e21d6064e50d5bb" }, { "dataPath": "params_shard_75.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.7.mlp.gate_up_proj.q_weight", "shape": [ 16384, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "060ba9a25db7e4dcfa972264cf1448a0" }, { "dataPath": "params_shard_76.bin", "format": "raw-shard", "nbytes": 22616064, "records": [ { "name": "transformer.h.7.mlp.down_proj.q_weight", "shape": [ 3072, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 0 }, { "name": "transformer.h.7.mlp.down_proj.q_scale", "shape": [ 3072, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 12582912 }, { "name": "transformer.h.7.mlp.gate_up_proj.q_scale", "shape": [ 16384, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 14155776 }, { "name": "transformer.h.7.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 17301504 }, { "name": "transformer.h.7.mixer.out_proj.q_weight", "shape": [ 3072, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 17307648 }, { "name": "transformer.h.7.mixer.out_proj.q_scale", "shape": [ 3072, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 22026240 } ], "md5sum": "4eb9955b9b5e4930dd379bdd0737a860" }, { "dataPath": "params_shard_77.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.8.mlp.gate_up_proj.q_weight", "shape": [ 16384, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "65b9831a3c07e0a5d2693ee65fc900d0" }, { "dataPath": "params_shard_78.bin", "format": "raw-shard", "nbytes": 33239040, "records": [ { "name": "transformer.h.7.mixer.qkv_proj.q_weight", "shape": [ 9216, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 0 }, { "name": "transformer.h.7.mixer.qkv_proj.q_scale", "shape": [ 9216, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1769472, "byteOffset": 14155776 }, { "name": "transformer.h.8.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 15925248 }, { "name": "transformer.h.8.mlp.down_proj.q_weight", "shape": [ 3072, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 15931392 }, { "name": "transformer.h.8.mlp.down_proj.q_scale", "shape": [ 3072, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 28514304 }, { "name": "transformer.h.8.mlp.gate_up_proj.q_scale", "shape": [ 16384, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 30087168 }, { "name": "transformer.h.8.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 33232896 } ], "md5sum": "3b5b808a1edf3ebb45c2e5c000396fb1" }, { "dataPath": "params_shard_79.bin", "format": "raw-shard", "nbytes": 21239808, "records": [ { "name": "transformer.h.8.mixer.out_proj.q_weight", "shape": [ 3072, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 0 }, { "name": "transformer.h.8.mixer.out_proj.q_scale", "shape": [ 3072, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 4718592 }, { "name": "transformer.h.8.mixer.qkv_proj.q_weight", "shape": [ 9216, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 5308416 }, { "name": "transformer.h.8.mixer.qkv_proj.q_scale", "shape": [ 9216, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1769472, "byteOffset": 19464192 }, { "name": "transformer.h.9.ln.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 21233664 } ], "md5sum": "f4acecbd4c88a0f5b6e91e5747b2f573" }, { "dataPath": "params_shard_80.bin", "format": "raw-shard", "nbytes": 25165824, "records": [ { "name": "transformer.h.9.mlp.gate_up_proj.q_weight", "shape": [ 16384, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25165824, "byteOffset": 0 } ], "md5sum": "6684c5aadf0f546b1bdec716959167e1" }, { "dataPath": "params_shard_81.bin", "format": "raw-shard", "nbytes": 22616064, "records": [ { "name": "transformer.h.9.mlp.down_proj.q_weight", "shape": [ 3072, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12582912, "byteOffset": 0 }, { "name": "transformer.h.9.mlp.down_proj.q_scale", "shape": [ 3072, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 12582912 }, { "name": "transformer.h.9.mlp.gate_up_proj.q_scale", "shape": [ 16384, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 14155776 }, { "name": "transformer.h.9.post_attention_layernorm.weight", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 17301504 }, { "name": "transformer.h.9.mixer.out_proj.q_weight", "shape": [ 3072, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 17307648 }, { "name": "transformer.h.9.mixer.out_proj.q_scale", "shape": [ 3072, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 22026240 } ], "md5sum": "ea57b18eac2ba6eb2f776685d5809f71" }, { "dataPath": "params_shard_82.bin", "format": "raw-shard", "nbytes": 15925248, "records": [ { "name": "transformer.h.9.mixer.qkv_proj.q_weight", "shape": [ 9216, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 14155776, "byteOffset": 0 }, { "name": "transformer.h.9.mixer.qkv_proj.q_scale", "shape": [ 9216, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1769472, "byteOffset": 14155776 } ], "md5sum": "865b527be9a21333488551e85239379f" } ] }