{ "schema": "trellis-webgpu-pack", "version": 1, "stage": "sparseStructureFlow", "upstreamRevision": "e0b00432b8e3a8ecee0df806ab1df9f7281f2be4", "shards": [ { "file": "stage-000.bin", "bytes": 66698240, "sha256": "336b12d08c0783f0c8c49b6ac077854b12b363d7014da5dfa10f8b9d898afcb6" }, { "file": "stage-001.bin", "bytes": 63298560, "sha256": "cf1a86dc354b2cc61cc5615b25ee6b7155ea0383505463b7c1494cd0873fd526" }, { "file": "stage-002.bin", "bytes": 65006080, "sha256": "0163964843d9069d9dd5a4957fc3687f020d1ace80c7dd3a6e2e404c7ed1796a" }, { "file": "stage-003.bin", "bytes": 66344960, "sha256": "e92160701cfa0a8a6a3aed0f306e990b75f87aad874367b8b922d5766b64aa3a" }, { "file": "stage-004.bin", "bytes": 61619200, "sha256": "11d63e67b9571c4921a3f7ebac12f91732c98342ba91acc034a9f852f50495b3" }, { "file": "stage-005.bin", "bytes": 66682880, "sha256": "23b9e6e81b60ab49dd41d5de52c857cbcca4d04eda85e08cb72627ab5fb90bfc" }, { "file": "stage-006.bin", "bytes": 63298560, "sha256": "2373d1cd96c902a88aaff969f7b948e3d2ce8df52fc649ae9e02b2785cc6e535" }, { "file": "stage-007.bin", "bytes": 65006080, "sha256": "750e19abf74ba1145bc6d267573dc9ef8b20cc9d169e2abf0ba3e64a78101e5c" }, { "file": "stage-008.bin", "bytes": 66344960, "sha256": "f2a212769aebc75fabce39b3d1696a663e9b61ca654d1d27bf560622d98dc915" }, { "file": "stage-009.bin", "bytes": 61619200, "sha256": "a5438a63aadcab2cf2f92f4a412a240e43208f4704c564b5407c57c88b3c52bd" }, { "file": "stage-010.bin", "bytes": 66682880, "sha256": "ceb66d586e63b40413741fabcc1c77f802fd235394f9573ca1307a92f61303fa" }, { "file": "stage-011.bin", "bytes": 63298560, "sha256": "adf005f79f4a15741ef02da4509fdb7cfe4b7cb9925a41b5fa0c0d374ff2a77c" }, { "file": "stage-012.bin", "bytes": 65006080, "sha256": "1ae821db1a432de2b0136cf20986f51f014062b2f5f3be5338e26aa1c195fc50" }, { "file": "stage-013.bin", "bytes": 66344960, "sha256": "37b9dcb08d2edb08d990234795c0d7a34b9f1b0180905d6a4b746a49fecac7d1" }, { "file": "stage-014.bin", "bytes": 61619200, "sha256": "e70537d509d17cfff3c6e2bff2e583f4298f716dd7fb2d13a92f0181bcdffdc6" }, { "file": "stage-015.bin", "bytes": 43416064, "sha256": "b7dea07abf9e37e9c01f76d735ef6a44d266dd913124918ccd28a70fdab22442" } ], "tensors": { "blocks.0.adaLN_modulation.1.bias": { "shape": [ 7680 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 0, "bytes": 15360 } }, "blocks.0.adaLN_modulation.1.weight": { "shape": [ 7680, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 7680, "columns": 1280, "padded_rows": 7680, "padded_columns": 1280, "data": { "shard": 0, "offset": 15360, "bytes": 9830400 }, "scales": { "shard": 0, "offset": 9845760, "bytes": 307200 } }, "blocks.0.cross_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 10152960, "bytes": 2560 } }, "blocks.0.cross_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 10155520, "bytes": 2560 } }, "blocks.0.cross_attn.to_kv.bias": { "shape": [ 2560 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 10158080, "bytes": 5120 } }, "blocks.0.cross_attn.to_kv.weight": { "shape": [ 2560, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2560, "columns": 768, "padded_rows": 2560, "padded_columns": 768, "data": { "shard": 0, "offset": 10163200, "bytes": 1966080 }, "scales": { "shard": 0, "offset": 12129280, "bytes": 61440 } }, "blocks.0.cross_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 12190720, "bytes": 2560 } }, "blocks.0.cross_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 0, "offset": 12193280, "bytes": 1638400 }, "scales": { "shard": 0, "offset": 13831680, "bytes": 51200 } }, "blocks.0.cross_attn.to_q.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 13882880, "bytes": 2560 } }, "blocks.0.cross_attn.to_q.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 0, "offset": 13885440, "bytes": 1638400 }, "scales": { "shard": 0, "offset": 15523840, "bytes": 51200 } }, "blocks.0.mlp.mlp.0.bias": { "shape": [ 5120 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 15575040, "bytes": 10240 } }, "blocks.0.mlp.mlp.0.weight": { "shape": [ 5120, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 5120, "columns": 1280, "padded_rows": 5120, "padded_columns": 1280, "data": { "shard": 0, "offset": 15585280, "bytes": 6553600 }, "scales": { "shard": 0, "offset": 22138880, "bytes": 204800 } }, "blocks.0.mlp.mlp.2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 22343680, "bytes": 2560 } }, "blocks.0.mlp.mlp.2.weight": { "shape": [ 1280, 5120 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 5120, "padded_rows": 1280, "padded_columns": 5120, "data": { "shard": 0, "offset": 22346240, "bytes": 6553600 }, "scales": { "shard": 0, "offset": 28899840, "bytes": 204800 } }, "blocks.0.norm2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 29104640, "bytes": 2560 } }, "blocks.0.norm2.weight": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 29107200, "bytes": 2560 } }, "blocks.0.self_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 29109760, "bytes": 2560 } }, "blocks.0.self_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 29112320, "bytes": 2560 } }, "blocks.0.self_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 29114880, "bytes": 2560 } }, "blocks.0.self_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 0, "offset": 29117440, "bytes": 1638400 }, "scales": { "shard": 0, "offset": 30755840, "bytes": 51200 } }, "blocks.0.self_attn.to_qkv.bias": { "shape": [ 3840 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 30807040, "bytes": 7680 } }, "blocks.0.self_attn.to_qkv.weight": { "shape": [ 3840, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3840, "columns": 1280, "padded_rows": 3840, "padded_columns": 1280, "data": { "shard": 0, "offset": 30814720, "bytes": 4915200 }, "scales": { "shard": 0, "offset": 35729920, "bytes": 153600 } }, "blocks.1.adaLN_modulation.1.bias": { "shape": [ 7680 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 35883520, "bytes": 15360 } }, "blocks.1.adaLN_modulation.1.weight": { "shape": [ 7680, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 7680, "columns": 1280, "padded_rows": 7680, "padded_columns": 1280, "data": { "shard": 0, "offset": 35898880, "bytes": 9830400 }, "scales": { "shard": 0, "offset": 45729280, "bytes": 307200 } }, "blocks.1.cross_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 46036480, "bytes": 2560 } }, "blocks.1.cross_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 46039040, "bytes": 2560 } }, "blocks.1.cross_attn.to_kv.bias": { "shape": [ 2560 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 46041600, "bytes": 5120 } }, "blocks.1.cross_attn.to_kv.weight": { "shape": [ 2560, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2560, "columns": 768, "padded_rows": 2560, "padded_columns": 768, "data": { "shard": 0, "offset": 46046720, "bytes": 1966080 }, "scales": { "shard": 0, "offset": 48012800, "bytes": 61440 } }, "blocks.1.cross_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 48074240, "bytes": 2560 } }, "blocks.1.cross_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 0, "offset": 48076800, "bytes": 1638400 }, "scales": { "shard": 0, "offset": 49715200, "bytes": 51200 } }, "blocks.1.cross_attn.to_q.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 49766400, "bytes": 2560 } }, "blocks.1.cross_attn.to_q.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 0, "offset": 49768960, "bytes": 1638400 }, "scales": { "shard": 0, "offset": 51407360, "bytes": 51200 } }, "blocks.1.mlp.mlp.0.bias": { "shape": [ 5120 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 51458560, "bytes": 10240 } }, "blocks.1.mlp.mlp.0.weight": { "shape": [ 5120, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 5120, "columns": 1280, "padded_rows": 5120, "padded_columns": 1280, "data": { "shard": 0, "offset": 51468800, "bytes": 6553600 }, "scales": { "shard": 0, "offset": 58022400, "bytes": 204800 } }, "blocks.1.mlp.mlp.2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 58227200, "bytes": 2560 } }, "blocks.1.mlp.mlp.2.weight": { "shape": [ 1280, 5120 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 5120, "padded_rows": 1280, "padded_columns": 5120, "data": { "shard": 0, "offset": 58229760, "bytes": 6553600 }, "scales": { "shard": 0, "offset": 64783360, "bytes": 204800 } }, "blocks.1.norm2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 64988160, "bytes": 2560 } }, "blocks.1.norm2.weight": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 64990720, "bytes": 2560 } }, "blocks.1.self_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 64993280, "bytes": 2560 } }, "blocks.1.self_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 64995840, "bytes": 2560 } }, "blocks.1.self_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 64998400, "bytes": 2560 } }, "blocks.1.self_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 0, "offset": 65000960, "bytes": 1638400 }, "scales": { "shard": 0, "offset": 66639360, "bytes": 51200 } }, "blocks.1.self_attn.to_qkv.bias": { "shape": [ 3840 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 66690560, "bytes": 7680 } }, "blocks.1.self_attn.to_qkv.weight": { "shape": [ 3840, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3840, "columns": 1280, "padded_rows": 3840, "padded_columns": 1280, "data": { "shard": 1, "offset": 0, "bytes": 4915200 }, "scales": { "shard": 1, "offset": 4915200, "bytes": 153600 } }, "blocks.10.adaLN_modulation.1.bias": { "shape": [ 7680 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 5068800, "bytes": 15360 } }, "blocks.10.adaLN_modulation.1.weight": { "shape": [ 7680, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 7680, "columns": 1280, "padded_rows": 7680, "padded_columns": 1280, "data": { "shard": 1, "offset": 5084160, "bytes": 9830400 }, "scales": { "shard": 1, "offset": 14914560, "bytes": 307200 } }, "blocks.10.cross_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 15221760, "bytes": 2560 } }, "blocks.10.cross_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 15224320, "bytes": 2560 } }, "blocks.10.cross_attn.to_kv.bias": { "shape": [ 2560 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 15226880, "bytes": 5120 } }, "blocks.10.cross_attn.to_kv.weight": { "shape": [ 2560, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2560, "columns": 768, "padded_rows": 2560, "padded_columns": 768, "data": { "shard": 1, "offset": 15232000, "bytes": 1966080 }, "scales": { "shard": 1, "offset": 17198080, "bytes": 61440 } }, "blocks.10.cross_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 17259520, "bytes": 2560 } }, "blocks.10.cross_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 1, "offset": 17262080, "bytes": 1638400 }, "scales": { "shard": 1, "offset": 18900480, "bytes": 51200 } }, "blocks.10.cross_attn.to_q.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 18951680, "bytes": 2560 } }, "blocks.10.cross_attn.to_q.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 1, "offset": 18954240, "bytes": 1638400 }, "scales": { "shard": 1, "offset": 20592640, "bytes": 51200 } }, "blocks.10.mlp.mlp.0.bias": { "shape": [ 5120 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 20643840, "bytes": 10240 } }, "blocks.10.mlp.mlp.0.weight": { "shape": [ 5120, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 5120, "columns": 1280, "padded_rows": 5120, "padded_columns": 1280, "data": { "shard": 1, "offset": 20654080, "bytes": 6553600 }, "scales": { "shard": 1, "offset": 27207680, "bytes": 204800 } }, "blocks.10.mlp.mlp.2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 27412480, "bytes": 2560 } }, "blocks.10.mlp.mlp.2.weight": { "shape": [ 1280, 5120 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 5120, "padded_rows": 1280, "padded_columns": 5120, "data": { "shard": 1, "offset": 27415040, "bytes": 6553600 }, "scales": { "shard": 1, "offset": 33968640, "bytes": 204800 } }, "blocks.10.norm2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 34173440, "bytes": 2560 } }, "blocks.10.norm2.weight": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 34176000, "bytes": 2560 } }, "blocks.10.self_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 34178560, "bytes": 2560 } }, "blocks.10.self_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 34181120, "bytes": 2560 } }, "blocks.10.self_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 34183680, "bytes": 2560 } }, "blocks.10.self_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 1, "offset": 34186240, "bytes": 1638400 }, "scales": { "shard": 1, "offset": 35824640, "bytes": 51200 } }, "blocks.10.self_attn.to_qkv.bias": { "shape": [ 3840 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 35875840, "bytes": 7680 } }, "blocks.10.self_attn.to_qkv.weight": { "shape": [ 3840, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3840, "columns": 1280, "padded_rows": 3840, "padded_columns": 1280, "data": { "shard": 1, "offset": 35883520, "bytes": 4915200 }, "scales": { "shard": 1, "offset": 40798720, "bytes": 153600 } }, "blocks.11.adaLN_modulation.1.bias": { "shape": [ 7680 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 40952320, "bytes": 15360 } }, "blocks.11.adaLN_modulation.1.weight": { "shape": [ 7680, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 7680, "columns": 1280, "padded_rows": 7680, "padded_columns": 1280, "data": { "shard": 1, "offset": 40967680, "bytes": 9830400 }, "scales": { "shard": 1, "offset": 50798080, "bytes": 307200 } }, "blocks.11.cross_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 51105280, "bytes": 2560 } }, "blocks.11.cross_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 51107840, "bytes": 2560 } }, "blocks.11.cross_attn.to_kv.bias": { "shape": [ 2560 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 51110400, "bytes": 5120 } }, "blocks.11.cross_attn.to_kv.weight": { "shape": [ 2560, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2560, "columns": 768, "padded_rows": 2560, "padded_columns": 768, "data": { "shard": 1, "offset": 51115520, "bytes": 1966080 }, "scales": { "shard": 1, "offset": 53081600, "bytes": 61440 } }, "blocks.11.cross_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 53143040, "bytes": 2560 } }, "blocks.11.cross_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 1, "offset": 53145600, "bytes": 1638400 }, "scales": { "shard": 1, "offset": 54784000, "bytes": 51200 } }, "blocks.11.cross_attn.to_q.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 54835200, "bytes": 2560 } }, "blocks.11.cross_attn.to_q.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 1, "offset": 54837760, "bytes": 1638400 }, "scales": { "shard": 1, "offset": 56476160, "bytes": 51200 } }, "blocks.11.mlp.mlp.0.bias": { "shape": [ 5120 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 56527360, "bytes": 10240 } }, "blocks.11.mlp.mlp.0.weight": { "shape": [ 5120, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 5120, "columns": 1280, "padded_rows": 5120, "padded_columns": 1280, "data": { "shard": 1, "offset": 56537600, "bytes": 6553600 }, "scales": { "shard": 1, "offset": 63091200, "bytes": 204800 } }, "blocks.11.mlp.mlp.2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 63296000, "bytes": 2560 } }, "blocks.11.mlp.mlp.2.weight": { "shape": [ 1280, 5120 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 5120, "padded_rows": 1280, "padded_columns": 5120, "data": { "shard": 2, "offset": 0, "bytes": 6553600 }, "scales": { "shard": 2, "offset": 6553600, "bytes": 204800 } }, "blocks.11.norm2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 6758400, "bytes": 2560 } }, "blocks.11.norm2.weight": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 6760960, "bytes": 2560 } }, "blocks.11.self_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 6763520, "bytes": 2560 } }, "blocks.11.self_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 6766080, "bytes": 2560 } }, "blocks.11.self_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 6768640, "bytes": 2560 } }, "blocks.11.self_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 2, "offset": 6771200, "bytes": 1638400 }, "scales": { "shard": 2, "offset": 8409600, "bytes": 51200 } }, "blocks.11.self_attn.to_qkv.bias": { "shape": [ 3840 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 8460800, "bytes": 7680 } }, "blocks.11.self_attn.to_qkv.weight": { "shape": [ 3840, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3840, "columns": 1280, "padded_rows": 3840, "padded_columns": 1280, "data": { "shard": 2, "offset": 8468480, "bytes": 4915200 }, "scales": { "shard": 2, "offset": 13383680, "bytes": 153600 } }, "blocks.12.adaLN_modulation.1.bias": { "shape": [ 7680 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 13537280, "bytes": 15360 } }, "blocks.12.adaLN_modulation.1.weight": { "shape": [ 7680, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 7680, "columns": 1280, "padded_rows": 7680, "padded_columns": 1280, "data": { "shard": 2, "offset": 13552640, "bytes": 9830400 }, "scales": { "shard": 2, "offset": 23383040, "bytes": 307200 } }, "blocks.12.cross_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 23690240, "bytes": 2560 } }, "blocks.12.cross_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 23692800, "bytes": 2560 } }, "blocks.12.cross_attn.to_kv.bias": { "shape": [ 2560 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 23695360, "bytes": 5120 } }, "blocks.12.cross_attn.to_kv.weight": { "shape": [ 2560, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2560, "columns": 768, "padded_rows": 2560, "padded_columns": 768, "data": { "shard": 2, "offset": 23700480, "bytes": 1966080 }, "scales": { "shard": 2, "offset": 25666560, "bytes": 61440 } }, "blocks.12.cross_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 25728000, "bytes": 2560 } }, "blocks.12.cross_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 2, "offset": 25730560, "bytes": 1638400 }, "scales": { "shard": 2, "offset": 27368960, "bytes": 51200 } }, "blocks.12.cross_attn.to_q.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 27420160, "bytes": 2560 } }, "blocks.12.cross_attn.to_q.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 2, "offset": 27422720, "bytes": 1638400 }, "scales": { "shard": 2, "offset": 29061120, "bytes": 51200 } }, "blocks.12.mlp.mlp.0.bias": { "shape": [ 5120 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 29112320, "bytes": 10240 } }, "blocks.12.mlp.mlp.0.weight": { "shape": [ 5120, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 5120, "columns": 1280, "padded_rows": 5120, "padded_columns": 1280, "data": { "shard": 2, "offset": 29122560, "bytes": 6553600 }, "scales": { "shard": 2, "offset": 35676160, "bytes": 204800 } }, "blocks.12.mlp.mlp.2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 35880960, "bytes": 2560 } }, "blocks.12.mlp.mlp.2.weight": { "shape": [ 1280, 5120 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 5120, "padded_rows": 1280, "padded_columns": 5120, "data": { "shard": 2, "offset": 35883520, "bytes": 6553600 }, "scales": { "shard": 2, "offset": 42437120, "bytes": 204800 } }, "blocks.12.norm2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 42641920, "bytes": 2560 } }, "blocks.12.norm2.weight": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 42644480, "bytes": 2560 } }, "blocks.12.self_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 42647040, "bytes": 2560 } }, "blocks.12.self_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 42649600, "bytes": 2560 } }, "blocks.12.self_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 42652160, "bytes": 2560 } }, "blocks.12.self_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 2, "offset": 42654720, "bytes": 1638400 }, "scales": { "shard": 2, "offset": 44293120, "bytes": 51200 } }, "blocks.12.self_attn.to_qkv.bias": { "shape": [ 3840 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 44344320, "bytes": 7680 } }, "blocks.12.self_attn.to_qkv.weight": { "shape": [ 3840, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3840, "columns": 1280, "padded_rows": 3840, "padded_columns": 1280, "data": { "shard": 2, "offset": 44352000, "bytes": 4915200 }, "scales": { "shard": 2, "offset": 49267200, "bytes": 153600 } }, "blocks.13.adaLN_modulation.1.bias": { "shape": [ 7680 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 49420800, "bytes": 15360 } }, "blocks.13.adaLN_modulation.1.weight": { "shape": [ 7680, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 7680, "columns": 1280, "padded_rows": 7680, "padded_columns": 1280, "data": { "shard": 2, "offset": 49436160, "bytes": 9830400 }, "scales": { "shard": 2, "offset": 59266560, "bytes": 307200 } }, "blocks.13.cross_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 59573760, "bytes": 2560 } }, "blocks.13.cross_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 59576320, "bytes": 2560 } }, "blocks.13.cross_attn.to_kv.bias": { "shape": [ 2560 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 59578880, "bytes": 5120 } }, "blocks.13.cross_attn.to_kv.weight": { "shape": [ 2560, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2560, "columns": 768, "padded_rows": 2560, "padded_columns": 768, "data": { "shard": 2, "offset": 59584000, "bytes": 1966080 }, "scales": { "shard": 2, "offset": 61550080, "bytes": 61440 } }, "blocks.13.cross_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 61611520, "bytes": 2560 } }, "blocks.13.cross_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 2, "offset": 61614080, "bytes": 1638400 }, "scales": { "shard": 2, "offset": 63252480, "bytes": 51200 } }, "blocks.13.cross_attn.to_q.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 63303680, "bytes": 2560 } }, "blocks.13.cross_attn.to_q.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 2, "offset": 63306240, "bytes": 1638400 }, "scales": { "shard": 2, "offset": 64944640, "bytes": 51200 } }, "blocks.13.mlp.mlp.0.bias": { "shape": [ 5120 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 64995840, "bytes": 10240 } }, "blocks.13.mlp.mlp.0.weight": { "shape": [ 5120, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 5120, "columns": 1280, "padded_rows": 5120, "padded_columns": 1280, "data": { "shard": 3, "offset": 0, "bytes": 6553600 }, "scales": { "shard": 3, "offset": 6553600, "bytes": 204800 } }, "blocks.13.mlp.mlp.2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 6758400, "bytes": 2560 } }, "blocks.13.mlp.mlp.2.weight": { "shape": [ 1280, 5120 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 5120, "padded_rows": 1280, "padded_columns": 5120, "data": { "shard": 3, "offset": 6760960, "bytes": 6553600 }, "scales": { "shard": 3, "offset": 13314560, "bytes": 204800 } }, "blocks.13.norm2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 13519360, "bytes": 2560 } }, "blocks.13.norm2.weight": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 13521920, "bytes": 2560 } }, "blocks.13.self_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 13524480, "bytes": 2560 } }, "blocks.13.self_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 13527040, "bytes": 2560 } }, "blocks.13.self_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 13529600, "bytes": 2560 } }, "blocks.13.self_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 3, "offset": 13532160, "bytes": 1638400 }, "scales": { "shard": 3, "offset": 15170560, "bytes": 51200 } }, "blocks.13.self_attn.to_qkv.bias": { "shape": [ 3840 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 15221760, "bytes": 7680 } }, "blocks.13.self_attn.to_qkv.weight": { "shape": [ 3840, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3840, "columns": 1280, "padded_rows": 3840, "padded_columns": 1280, "data": { "shard": 3, "offset": 15229440, "bytes": 4915200 }, "scales": { "shard": 3, "offset": 20144640, "bytes": 153600 } }, "blocks.14.adaLN_modulation.1.bias": { "shape": [ 7680 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 20298240, "bytes": 15360 } }, "blocks.14.adaLN_modulation.1.weight": { "shape": [ 7680, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 7680, "columns": 1280, "padded_rows": 7680, "padded_columns": 1280, "data": { "shard": 3, "offset": 20313600, "bytes": 9830400 }, "scales": { "shard": 3, "offset": 30144000, "bytes": 307200 } }, "blocks.14.cross_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 30451200, "bytes": 2560 } }, "blocks.14.cross_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 30453760, "bytes": 2560 } }, "blocks.14.cross_attn.to_kv.bias": { "shape": [ 2560 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 30456320, "bytes": 5120 } }, "blocks.14.cross_attn.to_kv.weight": { "shape": [ 2560, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2560, "columns": 768, "padded_rows": 2560, "padded_columns": 768, "data": { "shard": 3, "offset": 30461440, "bytes": 1966080 }, "scales": { "shard": 3, "offset": 32427520, "bytes": 61440 } }, "blocks.14.cross_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 32488960, "bytes": 2560 } }, "blocks.14.cross_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 3, "offset": 32491520, "bytes": 1638400 }, "scales": { "shard": 3, "offset": 34129920, "bytes": 51200 } }, "blocks.14.cross_attn.to_q.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 34181120, "bytes": 2560 } }, "blocks.14.cross_attn.to_q.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 3, "offset": 34183680, "bytes": 1638400 }, "scales": { "shard": 3, "offset": 35822080, "bytes": 51200 } }, "blocks.14.mlp.mlp.0.bias": { "shape": [ 5120 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 35873280, "bytes": 10240 } }, "blocks.14.mlp.mlp.0.weight": { "shape": [ 5120, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 5120, "columns": 1280, "padded_rows": 5120, "padded_columns": 1280, "data": { "shard": 3, "offset": 35883520, "bytes": 6553600 }, "scales": { "shard": 3, "offset": 42437120, "bytes": 204800 } }, "blocks.14.mlp.mlp.2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 42641920, "bytes": 2560 } }, "blocks.14.mlp.mlp.2.weight": { "shape": [ 1280, 5120 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 5120, "padded_rows": 1280, "padded_columns": 5120, "data": { "shard": 3, "offset": 42644480, "bytes": 6553600 }, "scales": { "shard": 3, "offset": 49198080, "bytes": 204800 } }, "blocks.14.norm2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 49402880, "bytes": 2560 } }, "blocks.14.norm2.weight": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 49405440, "bytes": 2560 } }, "blocks.14.self_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 49408000, "bytes": 2560 } }, "blocks.14.self_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 49410560, "bytes": 2560 } }, "blocks.14.self_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 49413120, "bytes": 2560 } }, "blocks.14.self_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 3, "offset": 49415680, "bytes": 1638400 }, "scales": { "shard": 3, "offset": 51054080, "bytes": 51200 } }, "blocks.14.self_attn.to_qkv.bias": { "shape": [ 3840 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 51105280, "bytes": 7680 } }, "blocks.14.self_attn.to_qkv.weight": { "shape": [ 3840, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3840, "columns": 1280, "padded_rows": 3840, "padded_columns": 1280, "data": { "shard": 3, "offset": 51112960, "bytes": 4915200 }, "scales": { "shard": 3, "offset": 56028160, "bytes": 153600 } }, "blocks.15.adaLN_modulation.1.bias": { "shape": [ 7680 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 56181760, "bytes": 15360 } }, "blocks.15.adaLN_modulation.1.weight": { "shape": [ 7680, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 7680, "columns": 1280, "padded_rows": 7680, "padded_columns": 1280, "data": { "shard": 3, "offset": 56197120, "bytes": 9830400 }, "scales": { "shard": 3, "offset": 66027520, "bytes": 307200 } }, "blocks.15.cross_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 66334720, "bytes": 2560 } }, "blocks.15.cross_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 66337280, "bytes": 2560 } }, "blocks.15.cross_attn.to_kv.bias": { "shape": [ 2560 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 66339840, "bytes": 5120 } }, "blocks.15.cross_attn.to_kv.weight": { "shape": [ 2560, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2560, "columns": 768, "padded_rows": 2560, "padded_columns": 768, "data": { "shard": 4, "offset": 0, "bytes": 1966080 }, "scales": { "shard": 4, "offset": 1966080, "bytes": 61440 } }, "blocks.15.cross_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 2027520, "bytes": 2560 } }, "blocks.15.cross_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 4, "offset": 2030080, "bytes": 1638400 }, "scales": { "shard": 4, "offset": 3668480, "bytes": 51200 } }, "blocks.15.cross_attn.to_q.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 3719680, "bytes": 2560 } }, "blocks.15.cross_attn.to_q.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 4, "offset": 3722240, "bytes": 1638400 }, "scales": { "shard": 4, "offset": 5360640, "bytes": 51200 } }, "blocks.15.mlp.mlp.0.bias": { "shape": [ 5120 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 5411840, "bytes": 10240 } }, "blocks.15.mlp.mlp.0.weight": { "shape": [ 5120, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 5120, "columns": 1280, "padded_rows": 5120, "padded_columns": 1280, "data": { "shard": 4, "offset": 5422080, "bytes": 6553600 }, "scales": { "shard": 4, "offset": 11975680, "bytes": 204800 } }, "blocks.15.mlp.mlp.2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 12180480, "bytes": 2560 } }, "blocks.15.mlp.mlp.2.weight": { "shape": [ 1280, 5120 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 5120, "padded_rows": 1280, "padded_columns": 5120, "data": { "shard": 4, "offset": 12183040, "bytes": 6553600 }, "scales": { "shard": 4, "offset": 18736640, "bytes": 204800 } }, "blocks.15.norm2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 18941440, "bytes": 2560 } }, "blocks.15.norm2.weight": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 18944000, "bytes": 2560 } }, "blocks.15.self_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 18946560, "bytes": 2560 } }, "blocks.15.self_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 18949120, "bytes": 2560 } }, "blocks.15.self_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 18951680, "bytes": 2560 } }, "blocks.15.self_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 4, "offset": 18954240, "bytes": 1638400 }, "scales": { "shard": 4, "offset": 20592640, "bytes": 51200 } }, "blocks.15.self_attn.to_qkv.bias": { "shape": [ 3840 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 20643840, "bytes": 7680 } }, "blocks.15.self_attn.to_qkv.weight": { "shape": [ 3840, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3840, "columns": 1280, "padded_rows": 3840, "padded_columns": 1280, "data": { "shard": 4, "offset": 20651520, "bytes": 4915200 }, "scales": { "shard": 4, "offset": 25566720, "bytes": 153600 } }, "blocks.16.adaLN_modulation.1.bias": { "shape": [ 7680 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 25720320, "bytes": 15360 } }, "blocks.16.adaLN_modulation.1.weight": { "shape": [ 7680, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 7680, "columns": 1280, "padded_rows": 7680, "padded_columns": 1280, "data": { "shard": 4, "offset": 25735680, "bytes": 9830400 }, "scales": { "shard": 4, "offset": 35566080, "bytes": 307200 } }, "blocks.16.cross_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 35873280, "bytes": 2560 } }, "blocks.16.cross_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 35875840, "bytes": 2560 } }, "blocks.16.cross_attn.to_kv.bias": { "shape": [ 2560 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 35878400, "bytes": 5120 } }, "blocks.16.cross_attn.to_kv.weight": { "shape": [ 2560, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2560, "columns": 768, "padded_rows": 2560, "padded_columns": 768, "data": { "shard": 4, "offset": 35883520, "bytes": 1966080 }, "scales": { "shard": 4, "offset": 37849600, "bytes": 61440 } }, "blocks.16.cross_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 37911040, "bytes": 2560 } }, "blocks.16.cross_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 4, "offset": 37913600, "bytes": 1638400 }, "scales": { "shard": 4, "offset": 39552000, "bytes": 51200 } }, "blocks.16.cross_attn.to_q.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 39603200, "bytes": 2560 } }, "blocks.16.cross_attn.to_q.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 4, "offset": 39605760, "bytes": 1638400 }, "scales": { "shard": 4, "offset": 41244160, "bytes": 51200 } }, "blocks.16.mlp.mlp.0.bias": { "shape": [ 5120 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 41295360, "bytes": 10240 } }, "blocks.16.mlp.mlp.0.weight": { "shape": [ 5120, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 5120, "columns": 1280, "padded_rows": 5120, "padded_columns": 1280, "data": { "shard": 4, "offset": 41305600, "bytes": 6553600 }, "scales": { "shard": 4, "offset": 47859200, "bytes": 204800 } }, "blocks.16.mlp.mlp.2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 48064000, "bytes": 2560 } }, "blocks.16.mlp.mlp.2.weight": { "shape": [ 1280, 5120 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 5120, "padded_rows": 1280, "padded_columns": 5120, "data": { "shard": 4, "offset": 48066560, "bytes": 6553600 }, "scales": { "shard": 4, "offset": 54620160, "bytes": 204800 } }, "blocks.16.norm2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 54824960, "bytes": 2560 } }, "blocks.16.norm2.weight": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 54827520, "bytes": 2560 } }, "blocks.16.self_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 54830080, "bytes": 2560 } }, "blocks.16.self_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 54832640, "bytes": 2560 } }, "blocks.16.self_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 54835200, "bytes": 2560 } }, "blocks.16.self_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 4, "offset": 54837760, "bytes": 1638400 }, "scales": { "shard": 4, "offset": 56476160, "bytes": 51200 } }, "blocks.16.self_attn.to_qkv.bias": { "shape": [ 3840 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 56527360, "bytes": 7680 } }, "blocks.16.self_attn.to_qkv.weight": { "shape": [ 3840, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3840, "columns": 1280, "padded_rows": 3840, "padded_columns": 1280, "data": { "shard": 4, "offset": 56535040, "bytes": 4915200 }, "scales": { "shard": 4, "offset": 61450240, "bytes": 153600 } }, "blocks.17.adaLN_modulation.1.bias": { "shape": [ 7680 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 61603840, "bytes": 15360 } }, "blocks.17.adaLN_modulation.1.weight": { "shape": [ 7680, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 7680, "columns": 1280, "padded_rows": 7680, "padded_columns": 1280, "data": { "shard": 5, "offset": 0, "bytes": 9830400 }, "scales": { "shard": 5, "offset": 9830400, "bytes": 307200 } }, "blocks.17.cross_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 10137600, "bytes": 2560 } }, "blocks.17.cross_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 10140160, "bytes": 2560 } }, "blocks.17.cross_attn.to_kv.bias": { "shape": [ 2560 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 10142720, "bytes": 5120 } }, "blocks.17.cross_attn.to_kv.weight": { "shape": [ 2560, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2560, "columns": 768, "padded_rows": 2560, "padded_columns": 768, "data": { "shard": 5, "offset": 10147840, "bytes": 1966080 }, "scales": { "shard": 5, "offset": 12113920, "bytes": 61440 } }, "blocks.17.cross_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 12175360, "bytes": 2560 } }, "blocks.17.cross_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 5, "offset": 12177920, "bytes": 1638400 }, "scales": { "shard": 5, "offset": 13816320, "bytes": 51200 } }, "blocks.17.cross_attn.to_q.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 13867520, "bytes": 2560 } }, "blocks.17.cross_attn.to_q.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 5, "offset": 13870080, "bytes": 1638400 }, "scales": { "shard": 5, "offset": 15508480, "bytes": 51200 } }, "blocks.17.mlp.mlp.0.bias": { "shape": [ 5120 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 15559680, "bytes": 10240 } }, "blocks.17.mlp.mlp.0.weight": { "shape": [ 5120, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 5120, "columns": 1280, "padded_rows": 5120, "padded_columns": 1280, "data": { "shard": 5, "offset": 15569920, "bytes": 6553600 }, "scales": { "shard": 5, "offset": 22123520, "bytes": 204800 } }, "blocks.17.mlp.mlp.2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 22328320, "bytes": 2560 } }, "blocks.17.mlp.mlp.2.weight": { "shape": [ 1280, 5120 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 5120, "padded_rows": 1280, "padded_columns": 5120, "data": { "shard": 5, "offset": 22330880, "bytes": 6553600 }, "scales": { "shard": 5, "offset": 28884480, "bytes": 204800 } }, "blocks.17.norm2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 29089280, "bytes": 2560 } }, "blocks.17.norm2.weight": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 29091840, "bytes": 2560 } }, "blocks.17.self_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 29094400, "bytes": 2560 } }, "blocks.17.self_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 29096960, "bytes": 2560 } }, "blocks.17.self_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 29099520, "bytes": 2560 } }, "blocks.17.self_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 5, "offset": 29102080, "bytes": 1638400 }, "scales": { "shard": 5, "offset": 30740480, "bytes": 51200 } }, "blocks.17.self_attn.to_qkv.bias": { "shape": [ 3840 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 30791680, "bytes": 7680 } }, "blocks.17.self_attn.to_qkv.weight": { "shape": [ 3840, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3840, "columns": 1280, "padded_rows": 3840, "padded_columns": 1280, "data": { "shard": 5, "offset": 30799360, "bytes": 4915200 }, "scales": { "shard": 5, "offset": 35714560, "bytes": 153600 } }, "blocks.18.adaLN_modulation.1.bias": { "shape": [ 7680 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 35868160, "bytes": 15360 } }, "blocks.18.adaLN_modulation.1.weight": { "shape": [ 7680, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 7680, "columns": 1280, "padded_rows": 7680, "padded_columns": 1280, "data": { "shard": 5, "offset": 35883520, "bytes": 9830400 }, "scales": { "shard": 5, "offset": 45713920, "bytes": 307200 } }, "blocks.18.cross_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 46021120, "bytes": 2560 } }, "blocks.18.cross_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 46023680, "bytes": 2560 } }, "blocks.18.cross_attn.to_kv.bias": { "shape": [ 2560 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 46026240, "bytes": 5120 } }, "blocks.18.cross_attn.to_kv.weight": { "shape": [ 2560, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2560, "columns": 768, "padded_rows": 2560, "padded_columns": 768, "data": { "shard": 5, "offset": 46031360, "bytes": 1966080 }, "scales": { "shard": 5, "offset": 47997440, "bytes": 61440 } }, "blocks.18.cross_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 48058880, "bytes": 2560 } }, "blocks.18.cross_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 5, "offset": 48061440, "bytes": 1638400 }, "scales": { "shard": 5, "offset": 49699840, "bytes": 51200 } }, "blocks.18.cross_attn.to_q.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 49751040, "bytes": 2560 } }, "blocks.18.cross_attn.to_q.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 5, "offset": 49753600, "bytes": 1638400 }, "scales": { "shard": 5, "offset": 51392000, "bytes": 51200 } }, "blocks.18.mlp.mlp.0.bias": { "shape": [ 5120 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 51443200, "bytes": 10240 } }, "blocks.18.mlp.mlp.0.weight": { "shape": [ 5120, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 5120, "columns": 1280, "padded_rows": 5120, "padded_columns": 1280, "data": { "shard": 5, "offset": 51453440, "bytes": 6553600 }, "scales": { "shard": 5, "offset": 58007040, "bytes": 204800 } }, "blocks.18.mlp.mlp.2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 58211840, "bytes": 2560 } }, "blocks.18.mlp.mlp.2.weight": { "shape": [ 1280, 5120 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 5120, "padded_rows": 1280, "padded_columns": 5120, "data": { "shard": 5, "offset": 58214400, "bytes": 6553600 }, "scales": { "shard": 5, "offset": 64768000, "bytes": 204800 } }, "blocks.18.norm2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 64972800, "bytes": 2560 } }, "blocks.18.norm2.weight": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 64975360, "bytes": 2560 } }, "blocks.18.self_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 64977920, "bytes": 2560 } }, "blocks.18.self_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 64980480, "bytes": 2560 } }, "blocks.18.self_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 64983040, "bytes": 2560 } }, "blocks.18.self_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 5, "offset": 64985600, "bytes": 1638400 }, "scales": { "shard": 5, "offset": 66624000, "bytes": 51200 } }, "blocks.18.self_attn.to_qkv.bias": { "shape": [ 3840 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 66675200, "bytes": 7680 } }, "blocks.18.self_attn.to_qkv.weight": { "shape": [ 3840, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3840, "columns": 1280, "padded_rows": 3840, "padded_columns": 1280, "data": { "shard": 6, "offset": 0, "bytes": 4915200 }, "scales": { "shard": 6, "offset": 4915200, "bytes": 153600 } }, "blocks.19.adaLN_modulation.1.bias": { "shape": [ 7680 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 5068800, "bytes": 15360 } }, "blocks.19.adaLN_modulation.1.weight": { "shape": [ 7680, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 7680, "columns": 1280, "padded_rows": 7680, "padded_columns": 1280, "data": { "shard": 6, "offset": 5084160, "bytes": 9830400 }, "scales": { "shard": 6, "offset": 14914560, "bytes": 307200 } }, "blocks.19.cross_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 15221760, "bytes": 2560 } }, "blocks.19.cross_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 15224320, "bytes": 2560 } }, "blocks.19.cross_attn.to_kv.bias": { "shape": [ 2560 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 15226880, "bytes": 5120 } }, "blocks.19.cross_attn.to_kv.weight": { "shape": [ 2560, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2560, "columns": 768, "padded_rows": 2560, "padded_columns": 768, "data": { "shard": 6, "offset": 15232000, "bytes": 1966080 }, "scales": { "shard": 6, "offset": 17198080, "bytes": 61440 } }, "blocks.19.cross_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 17259520, "bytes": 2560 } }, "blocks.19.cross_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 6, "offset": 17262080, "bytes": 1638400 }, "scales": { "shard": 6, "offset": 18900480, "bytes": 51200 } }, "blocks.19.cross_attn.to_q.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 18951680, "bytes": 2560 } }, "blocks.19.cross_attn.to_q.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 6, "offset": 18954240, "bytes": 1638400 }, "scales": { "shard": 6, "offset": 20592640, "bytes": 51200 } }, "blocks.19.mlp.mlp.0.bias": { "shape": [ 5120 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 20643840, "bytes": 10240 } }, "blocks.19.mlp.mlp.0.weight": { "shape": [ 5120, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 5120, "columns": 1280, "padded_rows": 5120, "padded_columns": 1280, "data": { "shard": 6, "offset": 20654080, "bytes": 6553600 }, "scales": { "shard": 6, "offset": 27207680, "bytes": 204800 } }, "blocks.19.mlp.mlp.2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 27412480, "bytes": 2560 } }, "blocks.19.mlp.mlp.2.weight": { "shape": [ 1280, 5120 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 5120, "padded_rows": 1280, "padded_columns": 5120, "data": { "shard": 6, "offset": 27415040, "bytes": 6553600 }, "scales": { "shard": 6, "offset": 33968640, "bytes": 204800 } }, "blocks.19.norm2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 34173440, "bytes": 2560 } }, "blocks.19.norm2.weight": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 34176000, "bytes": 2560 } }, "blocks.19.self_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 34178560, "bytes": 2560 } }, "blocks.19.self_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 34181120, "bytes": 2560 } }, "blocks.19.self_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 34183680, "bytes": 2560 } }, "blocks.19.self_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 6, "offset": 34186240, "bytes": 1638400 }, "scales": { "shard": 6, "offset": 35824640, "bytes": 51200 } }, "blocks.19.self_attn.to_qkv.bias": { "shape": [ 3840 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 35875840, "bytes": 7680 } }, "blocks.19.self_attn.to_qkv.weight": { "shape": [ 3840, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3840, "columns": 1280, "padded_rows": 3840, "padded_columns": 1280, "data": { "shard": 6, "offset": 35883520, "bytes": 4915200 }, "scales": { "shard": 6, "offset": 40798720, "bytes": 153600 } }, "blocks.2.adaLN_modulation.1.bias": { "shape": [ 7680 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 40952320, "bytes": 15360 } }, "blocks.2.adaLN_modulation.1.weight": { "shape": [ 7680, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 7680, "columns": 1280, "padded_rows": 7680, "padded_columns": 1280, "data": { "shard": 6, "offset": 40967680, "bytes": 9830400 }, "scales": { "shard": 6, "offset": 50798080, "bytes": 307200 } }, "blocks.2.cross_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 51105280, "bytes": 2560 } }, "blocks.2.cross_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 51107840, "bytes": 2560 } }, "blocks.2.cross_attn.to_kv.bias": { "shape": [ 2560 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 51110400, "bytes": 5120 } }, "blocks.2.cross_attn.to_kv.weight": { "shape": [ 2560, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2560, "columns": 768, "padded_rows": 2560, "padded_columns": 768, "data": { "shard": 6, "offset": 51115520, "bytes": 1966080 }, "scales": { "shard": 6, "offset": 53081600, "bytes": 61440 } }, "blocks.2.cross_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 53143040, "bytes": 2560 } }, "blocks.2.cross_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 6, "offset": 53145600, "bytes": 1638400 }, "scales": { "shard": 6, "offset": 54784000, "bytes": 51200 } }, "blocks.2.cross_attn.to_q.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 54835200, "bytes": 2560 } }, "blocks.2.cross_attn.to_q.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 6, "offset": 54837760, "bytes": 1638400 }, "scales": { "shard": 6, "offset": 56476160, "bytes": 51200 } }, "blocks.2.mlp.mlp.0.bias": { "shape": [ 5120 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 56527360, "bytes": 10240 } }, "blocks.2.mlp.mlp.0.weight": { "shape": [ 5120, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 5120, "columns": 1280, "padded_rows": 5120, "padded_columns": 1280, "data": { "shard": 6, "offset": 56537600, "bytes": 6553600 }, "scales": { "shard": 6, "offset": 63091200, "bytes": 204800 } }, "blocks.2.mlp.mlp.2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 63296000, "bytes": 2560 } }, "blocks.2.mlp.mlp.2.weight": { "shape": [ 1280, 5120 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 5120, "padded_rows": 1280, "padded_columns": 5120, "data": { "shard": 7, "offset": 0, "bytes": 6553600 }, "scales": { "shard": 7, "offset": 6553600, "bytes": 204800 } }, "blocks.2.norm2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 6758400, "bytes": 2560 } }, "blocks.2.norm2.weight": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 6760960, "bytes": 2560 } }, "blocks.2.self_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 6763520, "bytes": 2560 } }, "blocks.2.self_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 6766080, "bytes": 2560 } }, "blocks.2.self_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 6768640, "bytes": 2560 } }, "blocks.2.self_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 7, "offset": 6771200, "bytes": 1638400 }, "scales": { "shard": 7, "offset": 8409600, "bytes": 51200 } }, "blocks.2.self_attn.to_qkv.bias": { "shape": [ 3840 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 8460800, "bytes": 7680 } }, "blocks.2.self_attn.to_qkv.weight": { "shape": [ 3840, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3840, "columns": 1280, "padded_rows": 3840, "padded_columns": 1280, "data": { "shard": 7, "offset": 8468480, "bytes": 4915200 }, "scales": { "shard": 7, "offset": 13383680, "bytes": 153600 } }, "blocks.20.adaLN_modulation.1.bias": { "shape": [ 7680 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 13537280, "bytes": 15360 } }, "blocks.20.adaLN_modulation.1.weight": { "shape": [ 7680, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 7680, "columns": 1280, "padded_rows": 7680, "padded_columns": 1280, "data": { "shard": 7, "offset": 13552640, "bytes": 9830400 }, "scales": { "shard": 7, "offset": 23383040, "bytes": 307200 } }, "blocks.20.cross_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 23690240, "bytes": 2560 } }, "blocks.20.cross_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 23692800, "bytes": 2560 } }, "blocks.20.cross_attn.to_kv.bias": { "shape": [ 2560 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 23695360, "bytes": 5120 } }, "blocks.20.cross_attn.to_kv.weight": { "shape": [ 2560, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2560, "columns": 768, "padded_rows": 2560, "padded_columns": 768, "data": { "shard": 7, "offset": 23700480, "bytes": 1966080 }, "scales": { "shard": 7, "offset": 25666560, "bytes": 61440 } }, "blocks.20.cross_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 25728000, "bytes": 2560 } }, "blocks.20.cross_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 7, "offset": 25730560, "bytes": 1638400 }, "scales": { "shard": 7, "offset": 27368960, "bytes": 51200 } }, "blocks.20.cross_attn.to_q.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 27420160, "bytes": 2560 } }, "blocks.20.cross_attn.to_q.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 7, "offset": 27422720, "bytes": 1638400 }, "scales": { "shard": 7, "offset": 29061120, "bytes": 51200 } }, "blocks.20.mlp.mlp.0.bias": { "shape": [ 5120 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 29112320, "bytes": 10240 } }, "blocks.20.mlp.mlp.0.weight": { "shape": [ 5120, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 5120, "columns": 1280, "padded_rows": 5120, "padded_columns": 1280, "data": { "shard": 7, "offset": 29122560, "bytes": 6553600 }, "scales": { "shard": 7, "offset": 35676160, "bytes": 204800 } }, "blocks.20.mlp.mlp.2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 35880960, "bytes": 2560 } }, "blocks.20.mlp.mlp.2.weight": { "shape": [ 1280, 5120 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 5120, "padded_rows": 1280, "padded_columns": 5120, "data": { "shard": 7, "offset": 35883520, "bytes": 6553600 }, "scales": { "shard": 7, "offset": 42437120, "bytes": 204800 } }, "blocks.20.norm2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 42641920, "bytes": 2560 } }, "blocks.20.norm2.weight": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 42644480, "bytes": 2560 } }, "blocks.20.self_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 42647040, "bytes": 2560 } }, "blocks.20.self_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 42649600, "bytes": 2560 } }, "blocks.20.self_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 42652160, "bytes": 2560 } }, "blocks.20.self_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 7, "offset": 42654720, "bytes": 1638400 }, "scales": { "shard": 7, "offset": 44293120, "bytes": 51200 } }, "blocks.20.self_attn.to_qkv.bias": { "shape": [ 3840 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 44344320, "bytes": 7680 } }, "blocks.20.self_attn.to_qkv.weight": { "shape": [ 3840, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3840, "columns": 1280, "padded_rows": 3840, "padded_columns": 1280, "data": { "shard": 7, "offset": 44352000, "bytes": 4915200 }, "scales": { "shard": 7, "offset": 49267200, "bytes": 153600 } }, "blocks.21.adaLN_modulation.1.bias": { "shape": [ 7680 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 49420800, "bytes": 15360 } }, "blocks.21.adaLN_modulation.1.weight": { "shape": [ 7680, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 7680, "columns": 1280, "padded_rows": 7680, "padded_columns": 1280, "data": { "shard": 7, "offset": 49436160, "bytes": 9830400 }, "scales": { "shard": 7, "offset": 59266560, "bytes": 307200 } }, "blocks.21.cross_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 59573760, "bytes": 2560 } }, "blocks.21.cross_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 59576320, "bytes": 2560 } }, "blocks.21.cross_attn.to_kv.bias": { "shape": [ 2560 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 59578880, "bytes": 5120 } }, "blocks.21.cross_attn.to_kv.weight": { "shape": [ 2560, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2560, "columns": 768, "padded_rows": 2560, "padded_columns": 768, "data": { "shard": 7, "offset": 59584000, "bytes": 1966080 }, "scales": { "shard": 7, "offset": 61550080, "bytes": 61440 } }, "blocks.21.cross_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 61611520, "bytes": 2560 } }, "blocks.21.cross_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 7, "offset": 61614080, "bytes": 1638400 }, "scales": { "shard": 7, "offset": 63252480, "bytes": 51200 } }, "blocks.21.cross_attn.to_q.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 63303680, "bytes": 2560 } }, "blocks.21.cross_attn.to_q.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 7, "offset": 63306240, "bytes": 1638400 }, "scales": { "shard": 7, "offset": 64944640, "bytes": 51200 } }, "blocks.21.mlp.mlp.0.bias": { "shape": [ 5120 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 64995840, "bytes": 10240 } }, "blocks.21.mlp.mlp.0.weight": { "shape": [ 5120, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 5120, "columns": 1280, "padded_rows": 5120, "padded_columns": 1280, "data": { "shard": 8, "offset": 0, "bytes": 6553600 }, "scales": { "shard": 8, "offset": 6553600, "bytes": 204800 } }, "blocks.21.mlp.mlp.2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 6758400, "bytes": 2560 } }, "blocks.21.mlp.mlp.2.weight": { "shape": [ 1280, 5120 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 5120, "padded_rows": 1280, "padded_columns": 5120, "data": { "shard": 8, "offset": 6760960, "bytes": 6553600 }, "scales": { "shard": 8, "offset": 13314560, "bytes": 204800 } }, "blocks.21.norm2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 13519360, "bytes": 2560 } }, "blocks.21.norm2.weight": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 13521920, "bytes": 2560 } }, "blocks.21.self_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 13524480, "bytes": 2560 } }, "blocks.21.self_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 13527040, "bytes": 2560 } }, "blocks.21.self_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 13529600, "bytes": 2560 } }, "blocks.21.self_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 8, "offset": 13532160, "bytes": 1638400 }, "scales": { "shard": 8, "offset": 15170560, "bytes": 51200 } }, "blocks.21.self_attn.to_qkv.bias": { "shape": [ 3840 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 15221760, "bytes": 7680 } }, "blocks.21.self_attn.to_qkv.weight": { "shape": [ 3840, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3840, "columns": 1280, "padded_rows": 3840, "padded_columns": 1280, "data": { "shard": 8, "offset": 15229440, "bytes": 4915200 }, "scales": { "shard": 8, "offset": 20144640, "bytes": 153600 } }, "blocks.22.adaLN_modulation.1.bias": { "shape": [ 7680 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 20298240, "bytes": 15360 } }, "blocks.22.adaLN_modulation.1.weight": { "shape": [ 7680, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 7680, "columns": 1280, "padded_rows": 7680, "padded_columns": 1280, "data": { "shard": 8, "offset": 20313600, "bytes": 9830400 }, "scales": { "shard": 8, "offset": 30144000, "bytes": 307200 } }, "blocks.22.cross_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 30451200, "bytes": 2560 } }, "blocks.22.cross_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 30453760, "bytes": 2560 } }, "blocks.22.cross_attn.to_kv.bias": { "shape": [ 2560 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 30456320, "bytes": 5120 } }, "blocks.22.cross_attn.to_kv.weight": { "shape": [ 2560, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2560, "columns": 768, "padded_rows": 2560, "padded_columns": 768, "data": { "shard": 8, "offset": 30461440, "bytes": 1966080 }, "scales": { "shard": 8, "offset": 32427520, "bytes": 61440 } }, "blocks.22.cross_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 32488960, "bytes": 2560 } }, "blocks.22.cross_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 8, "offset": 32491520, "bytes": 1638400 }, "scales": { "shard": 8, "offset": 34129920, "bytes": 51200 } }, "blocks.22.cross_attn.to_q.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 34181120, "bytes": 2560 } }, "blocks.22.cross_attn.to_q.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 8, "offset": 34183680, "bytes": 1638400 }, "scales": { "shard": 8, "offset": 35822080, "bytes": 51200 } }, "blocks.22.mlp.mlp.0.bias": { "shape": [ 5120 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 35873280, "bytes": 10240 } }, "blocks.22.mlp.mlp.0.weight": { "shape": [ 5120, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 5120, "columns": 1280, "padded_rows": 5120, "padded_columns": 1280, "data": { "shard": 8, "offset": 35883520, "bytes": 6553600 }, "scales": { "shard": 8, "offset": 42437120, "bytes": 204800 } }, "blocks.22.mlp.mlp.2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 42641920, "bytes": 2560 } }, "blocks.22.mlp.mlp.2.weight": { "shape": [ 1280, 5120 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 5120, "padded_rows": 1280, "padded_columns": 5120, "data": { "shard": 8, "offset": 42644480, "bytes": 6553600 }, "scales": { "shard": 8, "offset": 49198080, "bytes": 204800 } }, "blocks.22.norm2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 49402880, "bytes": 2560 } }, "blocks.22.norm2.weight": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 49405440, "bytes": 2560 } }, "blocks.22.self_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 49408000, "bytes": 2560 } }, "blocks.22.self_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 49410560, "bytes": 2560 } }, "blocks.22.self_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 49413120, "bytes": 2560 } }, "blocks.22.self_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 8, "offset": 49415680, "bytes": 1638400 }, "scales": { "shard": 8, "offset": 51054080, "bytes": 51200 } }, "blocks.22.self_attn.to_qkv.bias": { "shape": [ 3840 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 51105280, "bytes": 7680 } }, "blocks.22.self_attn.to_qkv.weight": { "shape": [ 3840, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3840, "columns": 1280, "padded_rows": 3840, "padded_columns": 1280, "data": { "shard": 8, "offset": 51112960, "bytes": 4915200 }, "scales": { "shard": 8, "offset": 56028160, "bytes": 153600 } }, "blocks.23.adaLN_modulation.1.bias": { "shape": [ 7680 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 56181760, "bytes": 15360 } }, "blocks.23.adaLN_modulation.1.weight": { "shape": [ 7680, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 7680, "columns": 1280, "padded_rows": 7680, "padded_columns": 1280, "data": { "shard": 8, "offset": 56197120, "bytes": 9830400 }, "scales": { "shard": 8, "offset": 66027520, "bytes": 307200 } }, "blocks.23.cross_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 66334720, "bytes": 2560 } }, "blocks.23.cross_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 66337280, "bytes": 2560 } }, "blocks.23.cross_attn.to_kv.bias": { "shape": [ 2560 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 66339840, "bytes": 5120 } }, "blocks.23.cross_attn.to_kv.weight": { "shape": [ 2560, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2560, "columns": 768, "padded_rows": 2560, "padded_columns": 768, "data": { "shard": 9, "offset": 0, "bytes": 1966080 }, "scales": { "shard": 9, "offset": 1966080, "bytes": 61440 } }, "blocks.23.cross_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 9, "offset": 2027520, "bytes": 2560 } }, "blocks.23.cross_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 9, "offset": 2030080, "bytes": 1638400 }, "scales": { "shard": 9, "offset": 3668480, "bytes": 51200 } }, "blocks.23.cross_attn.to_q.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 9, "offset": 3719680, "bytes": 2560 } }, "blocks.23.cross_attn.to_q.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 9, "offset": 3722240, "bytes": 1638400 }, "scales": { "shard": 9, "offset": 5360640, "bytes": 51200 } }, "blocks.23.mlp.mlp.0.bias": { "shape": [ 5120 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 9, "offset": 5411840, "bytes": 10240 } }, "blocks.23.mlp.mlp.0.weight": { "shape": [ 5120, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 5120, "columns": 1280, "padded_rows": 5120, "padded_columns": 1280, "data": { "shard": 9, "offset": 5422080, "bytes": 6553600 }, "scales": { "shard": 9, "offset": 11975680, "bytes": 204800 } }, "blocks.23.mlp.mlp.2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 9, "offset": 12180480, "bytes": 2560 } }, "blocks.23.mlp.mlp.2.weight": { "shape": [ 1280, 5120 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 5120, "padded_rows": 1280, "padded_columns": 5120, "data": { "shard": 9, "offset": 12183040, "bytes": 6553600 }, "scales": { "shard": 9, "offset": 18736640, "bytes": 204800 } }, "blocks.23.norm2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 9, "offset": 18941440, "bytes": 2560 } }, "blocks.23.norm2.weight": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 9, "offset": 18944000, "bytes": 2560 } }, "blocks.23.self_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 9, "offset": 18946560, "bytes": 2560 } }, "blocks.23.self_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 9, "offset": 18949120, "bytes": 2560 } }, "blocks.23.self_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 9, "offset": 18951680, "bytes": 2560 } }, "blocks.23.self_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 9, "offset": 18954240, "bytes": 1638400 }, "scales": { "shard": 9, "offset": 20592640, "bytes": 51200 } }, "blocks.23.self_attn.to_qkv.bias": { "shape": [ 3840 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 9, "offset": 20643840, "bytes": 7680 } }, "blocks.23.self_attn.to_qkv.weight": { "shape": [ 3840, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3840, "columns": 1280, "padded_rows": 3840, "padded_columns": 1280, "data": { "shard": 9, "offset": 20651520, "bytes": 4915200 }, "scales": { "shard": 9, "offset": 25566720, "bytes": 153600 } }, "blocks.24.adaLN_modulation.1.bias": { "shape": [ 7680 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 9, "offset": 25720320, "bytes": 15360 } }, "blocks.24.adaLN_modulation.1.weight": { "shape": [ 7680, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 7680, "columns": 1280, "padded_rows": 7680, "padded_columns": 1280, "data": { "shard": 9, "offset": 25735680, "bytes": 9830400 }, "scales": { "shard": 9, "offset": 35566080, "bytes": 307200 } }, "blocks.24.cross_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 9, "offset": 35873280, "bytes": 2560 } }, "blocks.24.cross_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 9, "offset": 35875840, "bytes": 2560 } }, "blocks.24.cross_attn.to_kv.bias": { "shape": [ 2560 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 9, "offset": 35878400, "bytes": 5120 } }, "blocks.24.cross_attn.to_kv.weight": { "shape": [ 2560, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2560, "columns": 768, "padded_rows": 2560, "padded_columns": 768, "data": { "shard": 9, "offset": 35883520, "bytes": 1966080 }, "scales": { "shard": 9, "offset": 37849600, "bytes": 61440 } }, "blocks.24.cross_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 9, "offset": 37911040, "bytes": 2560 } }, "blocks.24.cross_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 9, "offset": 37913600, "bytes": 1638400 }, "scales": { "shard": 9, "offset": 39552000, "bytes": 51200 } }, "blocks.24.cross_attn.to_q.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 9, "offset": 39603200, "bytes": 2560 } }, "blocks.24.cross_attn.to_q.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 9, "offset": 39605760, "bytes": 1638400 }, "scales": { "shard": 9, "offset": 41244160, "bytes": 51200 } }, "blocks.24.mlp.mlp.0.bias": { "shape": [ 5120 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 9, "offset": 41295360, "bytes": 10240 } }, "blocks.24.mlp.mlp.0.weight": { "shape": [ 5120, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 5120, "columns": 1280, "padded_rows": 5120, "padded_columns": 1280, "data": { "shard": 9, "offset": 41305600, "bytes": 6553600 }, "scales": { "shard": 9, "offset": 47859200, "bytes": 204800 } }, "blocks.24.mlp.mlp.2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 9, "offset": 48064000, "bytes": 2560 } }, "blocks.24.mlp.mlp.2.weight": { "shape": [ 1280, 5120 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 5120, "padded_rows": 1280, "padded_columns": 5120, "data": { "shard": 9, "offset": 48066560, "bytes": 6553600 }, "scales": { "shard": 9, "offset": 54620160, "bytes": 204800 } }, "blocks.24.norm2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 9, "offset": 54824960, "bytes": 2560 } }, "blocks.24.norm2.weight": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 9, "offset": 54827520, "bytes": 2560 } }, "blocks.24.self_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 9, "offset": 54830080, "bytes": 2560 } }, "blocks.24.self_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 9, "offset": 54832640, "bytes": 2560 } }, "blocks.24.self_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 9, "offset": 54835200, "bytes": 2560 } }, "blocks.24.self_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 9, "offset": 54837760, "bytes": 1638400 }, "scales": { "shard": 9, "offset": 56476160, "bytes": 51200 } }, "blocks.24.self_attn.to_qkv.bias": { "shape": [ 3840 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 9, "offset": 56527360, "bytes": 7680 } }, "blocks.24.self_attn.to_qkv.weight": { "shape": [ 3840, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3840, "columns": 1280, "padded_rows": 3840, "padded_columns": 1280, "data": { "shard": 9, "offset": 56535040, "bytes": 4915200 }, "scales": { "shard": 9, "offset": 61450240, "bytes": 153600 } }, "blocks.25.adaLN_modulation.1.bias": { "shape": [ 7680 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 9, "offset": 61603840, "bytes": 15360 } }, "blocks.25.adaLN_modulation.1.weight": { "shape": [ 7680, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 7680, "columns": 1280, "padded_rows": 7680, "padded_columns": 1280, "data": { "shard": 10, "offset": 0, "bytes": 9830400 }, "scales": { "shard": 10, "offset": 9830400, "bytes": 307200 } }, "blocks.25.cross_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 10, "offset": 10137600, "bytes": 2560 } }, "blocks.25.cross_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 10, "offset": 10140160, "bytes": 2560 } }, "blocks.25.cross_attn.to_kv.bias": { "shape": [ 2560 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 10, "offset": 10142720, "bytes": 5120 } }, "blocks.25.cross_attn.to_kv.weight": { "shape": [ 2560, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2560, "columns": 768, "padded_rows": 2560, "padded_columns": 768, "data": { "shard": 10, "offset": 10147840, "bytes": 1966080 }, "scales": { "shard": 10, "offset": 12113920, "bytes": 61440 } }, "blocks.25.cross_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 10, "offset": 12175360, "bytes": 2560 } }, "blocks.25.cross_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 10, "offset": 12177920, "bytes": 1638400 }, "scales": { "shard": 10, "offset": 13816320, "bytes": 51200 } }, "blocks.25.cross_attn.to_q.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 10, "offset": 13867520, "bytes": 2560 } }, "blocks.25.cross_attn.to_q.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 10, "offset": 13870080, "bytes": 1638400 }, "scales": { "shard": 10, "offset": 15508480, "bytes": 51200 } }, "blocks.25.mlp.mlp.0.bias": { "shape": [ 5120 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 10, "offset": 15559680, "bytes": 10240 } }, "blocks.25.mlp.mlp.0.weight": { "shape": [ 5120, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 5120, "columns": 1280, "padded_rows": 5120, "padded_columns": 1280, "data": { "shard": 10, "offset": 15569920, "bytes": 6553600 }, "scales": { "shard": 10, "offset": 22123520, "bytes": 204800 } }, "blocks.25.mlp.mlp.2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 10, "offset": 22328320, "bytes": 2560 } }, "blocks.25.mlp.mlp.2.weight": { "shape": [ 1280, 5120 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 5120, "padded_rows": 1280, "padded_columns": 5120, "data": { "shard": 10, "offset": 22330880, "bytes": 6553600 }, "scales": { "shard": 10, "offset": 28884480, "bytes": 204800 } }, "blocks.25.norm2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 10, "offset": 29089280, "bytes": 2560 } }, "blocks.25.norm2.weight": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 10, "offset": 29091840, "bytes": 2560 } }, "blocks.25.self_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 10, "offset": 29094400, "bytes": 2560 } }, "blocks.25.self_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 10, "offset": 29096960, "bytes": 2560 } }, "blocks.25.self_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 10, "offset": 29099520, "bytes": 2560 } }, "blocks.25.self_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 10, "offset": 29102080, "bytes": 1638400 }, "scales": { "shard": 10, "offset": 30740480, "bytes": 51200 } }, "blocks.25.self_attn.to_qkv.bias": { "shape": [ 3840 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 10, "offset": 30791680, "bytes": 7680 } }, "blocks.25.self_attn.to_qkv.weight": { "shape": [ 3840, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3840, "columns": 1280, "padded_rows": 3840, "padded_columns": 1280, "data": { "shard": 10, "offset": 30799360, "bytes": 4915200 }, "scales": { "shard": 10, "offset": 35714560, "bytes": 153600 } }, "blocks.26.adaLN_modulation.1.bias": { "shape": [ 7680 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 10, "offset": 35868160, "bytes": 15360 } }, "blocks.26.adaLN_modulation.1.weight": { "shape": [ 7680, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 7680, "columns": 1280, "padded_rows": 7680, "padded_columns": 1280, "data": { "shard": 10, "offset": 35883520, "bytes": 9830400 }, "scales": { "shard": 10, "offset": 45713920, "bytes": 307200 } }, "blocks.26.cross_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 10, "offset": 46021120, "bytes": 2560 } }, "blocks.26.cross_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 10, "offset": 46023680, "bytes": 2560 } }, "blocks.26.cross_attn.to_kv.bias": { "shape": [ 2560 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 10, "offset": 46026240, "bytes": 5120 } }, "blocks.26.cross_attn.to_kv.weight": { "shape": [ 2560, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2560, "columns": 768, "padded_rows": 2560, "padded_columns": 768, "data": { "shard": 10, "offset": 46031360, "bytes": 1966080 }, "scales": { "shard": 10, "offset": 47997440, "bytes": 61440 } }, "blocks.26.cross_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 10, "offset": 48058880, "bytes": 2560 } }, "blocks.26.cross_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 10, "offset": 48061440, "bytes": 1638400 }, "scales": { "shard": 10, "offset": 49699840, "bytes": 51200 } }, "blocks.26.cross_attn.to_q.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 10, "offset": 49751040, "bytes": 2560 } }, "blocks.26.cross_attn.to_q.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 10, "offset": 49753600, "bytes": 1638400 }, "scales": { "shard": 10, "offset": 51392000, "bytes": 51200 } }, "blocks.26.mlp.mlp.0.bias": { "shape": [ 5120 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 10, "offset": 51443200, "bytes": 10240 } }, "blocks.26.mlp.mlp.0.weight": { "shape": [ 5120, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 5120, "columns": 1280, "padded_rows": 5120, "padded_columns": 1280, "data": { "shard": 10, "offset": 51453440, "bytes": 6553600 }, "scales": { "shard": 10, "offset": 58007040, "bytes": 204800 } }, "blocks.26.mlp.mlp.2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 10, "offset": 58211840, "bytes": 2560 } }, "blocks.26.mlp.mlp.2.weight": { "shape": [ 1280, 5120 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 5120, "padded_rows": 1280, "padded_columns": 5120, "data": { "shard": 10, "offset": 58214400, "bytes": 6553600 }, "scales": { "shard": 10, "offset": 64768000, "bytes": 204800 } }, "blocks.26.norm2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 10, "offset": 64972800, "bytes": 2560 } }, "blocks.26.norm2.weight": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 10, "offset": 64975360, "bytes": 2560 } }, "blocks.26.self_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 10, "offset": 64977920, "bytes": 2560 } }, "blocks.26.self_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 10, "offset": 64980480, "bytes": 2560 } }, "blocks.26.self_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 10, "offset": 64983040, "bytes": 2560 } }, "blocks.26.self_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 10, "offset": 64985600, "bytes": 1638400 }, "scales": { "shard": 10, "offset": 66624000, "bytes": 51200 } }, "blocks.26.self_attn.to_qkv.bias": { "shape": [ 3840 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 10, "offset": 66675200, "bytes": 7680 } }, "blocks.26.self_attn.to_qkv.weight": { "shape": [ 3840, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3840, "columns": 1280, "padded_rows": 3840, "padded_columns": 1280, "data": { "shard": 11, "offset": 0, "bytes": 4915200 }, "scales": { "shard": 11, "offset": 4915200, "bytes": 153600 } }, "blocks.27.adaLN_modulation.1.bias": { "shape": [ 7680 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 11, "offset": 5068800, "bytes": 15360 } }, "blocks.27.adaLN_modulation.1.weight": { "shape": [ 7680, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 7680, "columns": 1280, "padded_rows": 7680, "padded_columns": 1280, "data": { "shard": 11, "offset": 5084160, "bytes": 9830400 }, "scales": { "shard": 11, "offset": 14914560, "bytes": 307200 } }, "blocks.27.cross_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 11, "offset": 15221760, "bytes": 2560 } }, "blocks.27.cross_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 11, "offset": 15224320, "bytes": 2560 } }, "blocks.27.cross_attn.to_kv.bias": { "shape": [ 2560 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 11, "offset": 15226880, "bytes": 5120 } }, "blocks.27.cross_attn.to_kv.weight": { "shape": [ 2560, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2560, "columns": 768, "padded_rows": 2560, "padded_columns": 768, "data": { "shard": 11, "offset": 15232000, "bytes": 1966080 }, "scales": { "shard": 11, "offset": 17198080, "bytes": 61440 } }, "blocks.27.cross_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 11, "offset": 17259520, "bytes": 2560 } }, "blocks.27.cross_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 11, "offset": 17262080, "bytes": 1638400 }, "scales": { "shard": 11, "offset": 18900480, "bytes": 51200 } }, "blocks.27.cross_attn.to_q.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 11, "offset": 18951680, "bytes": 2560 } }, "blocks.27.cross_attn.to_q.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 11, "offset": 18954240, "bytes": 1638400 }, "scales": { "shard": 11, "offset": 20592640, "bytes": 51200 } }, "blocks.27.mlp.mlp.0.bias": { "shape": [ 5120 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 11, "offset": 20643840, "bytes": 10240 } }, "blocks.27.mlp.mlp.0.weight": { "shape": [ 5120, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 5120, "columns": 1280, "padded_rows": 5120, "padded_columns": 1280, "data": { "shard": 11, "offset": 20654080, "bytes": 6553600 }, "scales": { "shard": 11, "offset": 27207680, "bytes": 204800 } }, "blocks.27.mlp.mlp.2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 11, "offset": 27412480, "bytes": 2560 } }, "blocks.27.mlp.mlp.2.weight": { "shape": [ 1280, 5120 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 5120, "padded_rows": 1280, "padded_columns": 5120, "data": { "shard": 11, "offset": 27415040, "bytes": 6553600 }, "scales": { "shard": 11, "offset": 33968640, "bytes": 204800 } }, "blocks.27.norm2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 11, "offset": 34173440, "bytes": 2560 } }, "blocks.27.norm2.weight": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 11, "offset": 34176000, "bytes": 2560 } }, "blocks.27.self_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 11, "offset": 34178560, "bytes": 2560 } }, "blocks.27.self_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 11, "offset": 34181120, "bytes": 2560 } }, "blocks.27.self_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 11, "offset": 34183680, "bytes": 2560 } }, "blocks.27.self_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 11, "offset": 34186240, "bytes": 1638400 }, "scales": { "shard": 11, "offset": 35824640, "bytes": 51200 } }, "blocks.27.self_attn.to_qkv.bias": { "shape": [ 3840 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 11, "offset": 35875840, "bytes": 7680 } }, "blocks.27.self_attn.to_qkv.weight": { "shape": [ 3840, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3840, "columns": 1280, "padded_rows": 3840, "padded_columns": 1280, "data": { "shard": 11, "offset": 35883520, "bytes": 4915200 }, "scales": { "shard": 11, "offset": 40798720, "bytes": 153600 } }, "blocks.3.adaLN_modulation.1.bias": { "shape": [ 7680 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 11, "offset": 40952320, "bytes": 15360 } }, "blocks.3.adaLN_modulation.1.weight": { "shape": [ 7680, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 7680, "columns": 1280, "padded_rows": 7680, "padded_columns": 1280, "data": { "shard": 11, "offset": 40967680, "bytes": 9830400 }, "scales": { "shard": 11, "offset": 50798080, "bytes": 307200 } }, "blocks.3.cross_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 11, "offset": 51105280, "bytes": 2560 } }, "blocks.3.cross_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 11, "offset": 51107840, "bytes": 2560 } }, "blocks.3.cross_attn.to_kv.bias": { "shape": [ 2560 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 11, "offset": 51110400, "bytes": 5120 } }, "blocks.3.cross_attn.to_kv.weight": { "shape": [ 2560, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2560, "columns": 768, "padded_rows": 2560, "padded_columns": 768, "data": { "shard": 11, "offset": 51115520, "bytes": 1966080 }, "scales": { "shard": 11, "offset": 53081600, "bytes": 61440 } }, "blocks.3.cross_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 11, "offset": 53143040, "bytes": 2560 } }, "blocks.3.cross_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 11, "offset": 53145600, "bytes": 1638400 }, "scales": { "shard": 11, "offset": 54784000, "bytes": 51200 } }, "blocks.3.cross_attn.to_q.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 11, "offset": 54835200, "bytes": 2560 } }, "blocks.3.cross_attn.to_q.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 11, "offset": 54837760, "bytes": 1638400 }, "scales": { "shard": 11, "offset": 56476160, "bytes": 51200 } }, "blocks.3.mlp.mlp.0.bias": { "shape": [ 5120 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 11, "offset": 56527360, "bytes": 10240 } }, "blocks.3.mlp.mlp.0.weight": { "shape": [ 5120, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 5120, "columns": 1280, "padded_rows": 5120, "padded_columns": 1280, "data": { "shard": 11, "offset": 56537600, "bytes": 6553600 }, "scales": { "shard": 11, "offset": 63091200, "bytes": 204800 } }, "blocks.3.mlp.mlp.2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 11, "offset": 63296000, "bytes": 2560 } }, "blocks.3.mlp.mlp.2.weight": { "shape": [ 1280, 5120 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 5120, "padded_rows": 1280, "padded_columns": 5120, "data": { "shard": 12, "offset": 0, "bytes": 6553600 }, "scales": { "shard": 12, "offset": 6553600, "bytes": 204800 } }, "blocks.3.norm2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 12, "offset": 6758400, "bytes": 2560 } }, "blocks.3.norm2.weight": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 12, "offset": 6760960, "bytes": 2560 } }, "blocks.3.self_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 12, "offset": 6763520, "bytes": 2560 } }, "blocks.3.self_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 12, "offset": 6766080, "bytes": 2560 } }, "blocks.3.self_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 12, "offset": 6768640, "bytes": 2560 } }, "blocks.3.self_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 12, "offset": 6771200, "bytes": 1638400 }, "scales": { "shard": 12, "offset": 8409600, "bytes": 51200 } }, "blocks.3.self_attn.to_qkv.bias": { "shape": [ 3840 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 12, "offset": 8460800, "bytes": 7680 } }, "blocks.3.self_attn.to_qkv.weight": { "shape": [ 3840, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3840, "columns": 1280, "padded_rows": 3840, "padded_columns": 1280, "data": { "shard": 12, "offset": 8468480, "bytes": 4915200 }, "scales": { "shard": 12, "offset": 13383680, "bytes": 153600 } }, "blocks.4.adaLN_modulation.1.bias": { "shape": [ 7680 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 12, "offset": 13537280, "bytes": 15360 } }, "blocks.4.adaLN_modulation.1.weight": { "shape": [ 7680, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 7680, "columns": 1280, "padded_rows": 7680, "padded_columns": 1280, "data": { "shard": 12, "offset": 13552640, "bytes": 9830400 }, "scales": { "shard": 12, "offset": 23383040, "bytes": 307200 } }, "blocks.4.cross_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 12, "offset": 23690240, "bytes": 2560 } }, "blocks.4.cross_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 12, "offset": 23692800, "bytes": 2560 } }, "blocks.4.cross_attn.to_kv.bias": { "shape": [ 2560 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 12, "offset": 23695360, "bytes": 5120 } }, "blocks.4.cross_attn.to_kv.weight": { "shape": [ 2560, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2560, "columns": 768, "padded_rows": 2560, "padded_columns": 768, "data": { "shard": 12, "offset": 23700480, "bytes": 1966080 }, "scales": { "shard": 12, "offset": 25666560, "bytes": 61440 } }, "blocks.4.cross_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 12, "offset": 25728000, "bytes": 2560 } }, "blocks.4.cross_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 12, "offset": 25730560, "bytes": 1638400 }, "scales": { "shard": 12, "offset": 27368960, "bytes": 51200 } }, "blocks.4.cross_attn.to_q.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 12, "offset": 27420160, "bytes": 2560 } }, "blocks.4.cross_attn.to_q.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 12, "offset": 27422720, "bytes": 1638400 }, "scales": { "shard": 12, "offset": 29061120, "bytes": 51200 } }, "blocks.4.mlp.mlp.0.bias": { "shape": [ 5120 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 12, "offset": 29112320, "bytes": 10240 } }, "blocks.4.mlp.mlp.0.weight": { "shape": [ 5120, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 5120, "columns": 1280, "padded_rows": 5120, "padded_columns": 1280, "data": { "shard": 12, "offset": 29122560, "bytes": 6553600 }, "scales": { "shard": 12, "offset": 35676160, "bytes": 204800 } }, "blocks.4.mlp.mlp.2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 12, "offset": 35880960, "bytes": 2560 } }, "blocks.4.mlp.mlp.2.weight": { "shape": [ 1280, 5120 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 5120, "padded_rows": 1280, "padded_columns": 5120, "data": { "shard": 12, "offset": 35883520, "bytes": 6553600 }, "scales": { "shard": 12, "offset": 42437120, "bytes": 204800 } }, "blocks.4.norm2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 12, "offset": 42641920, "bytes": 2560 } }, "blocks.4.norm2.weight": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 12, "offset": 42644480, "bytes": 2560 } }, "blocks.4.self_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 12, "offset": 42647040, "bytes": 2560 } }, "blocks.4.self_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 12, "offset": 42649600, "bytes": 2560 } }, "blocks.4.self_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 12, "offset": 42652160, "bytes": 2560 } }, "blocks.4.self_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 12, "offset": 42654720, "bytes": 1638400 }, "scales": { "shard": 12, "offset": 44293120, "bytes": 51200 } }, "blocks.4.self_attn.to_qkv.bias": { "shape": [ 3840 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 12, "offset": 44344320, "bytes": 7680 } }, "blocks.4.self_attn.to_qkv.weight": { "shape": [ 3840, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3840, "columns": 1280, "padded_rows": 3840, "padded_columns": 1280, "data": { "shard": 12, "offset": 44352000, "bytes": 4915200 }, "scales": { "shard": 12, "offset": 49267200, "bytes": 153600 } }, "blocks.5.adaLN_modulation.1.bias": { "shape": [ 7680 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 12, "offset": 49420800, "bytes": 15360 } }, "blocks.5.adaLN_modulation.1.weight": { "shape": [ 7680, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 7680, "columns": 1280, "padded_rows": 7680, "padded_columns": 1280, "data": { "shard": 12, "offset": 49436160, "bytes": 9830400 }, "scales": { "shard": 12, "offset": 59266560, "bytes": 307200 } }, "blocks.5.cross_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 12, "offset": 59573760, "bytes": 2560 } }, "blocks.5.cross_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 12, "offset": 59576320, "bytes": 2560 } }, "blocks.5.cross_attn.to_kv.bias": { "shape": [ 2560 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 12, "offset": 59578880, "bytes": 5120 } }, "blocks.5.cross_attn.to_kv.weight": { "shape": [ 2560, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2560, "columns": 768, "padded_rows": 2560, "padded_columns": 768, "data": { "shard": 12, "offset": 59584000, "bytes": 1966080 }, "scales": { "shard": 12, "offset": 61550080, "bytes": 61440 } }, "blocks.5.cross_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 12, "offset": 61611520, "bytes": 2560 } }, "blocks.5.cross_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 12, "offset": 61614080, "bytes": 1638400 }, "scales": { "shard": 12, "offset": 63252480, "bytes": 51200 } }, "blocks.5.cross_attn.to_q.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 12, "offset": 63303680, "bytes": 2560 } }, "blocks.5.cross_attn.to_q.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 12, "offset": 63306240, "bytes": 1638400 }, "scales": { "shard": 12, "offset": 64944640, "bytes": 51200 } }, "blocks.5.mlp.mlp.0.bias": { "shape": [ 5120 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 12, "offset": 64995840, "bytes": 10240 } }, "blocks.5.mlp.mlp.0.weight": { "shape": [ 5120, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 5120, "columns": 1280, "padded_rows": 5120, "padded_columns": 1280, "data": { "shard": 13, "offset": 0, "bytes": 6553600 }, "scales": { "shard": 13, "offset": 6553600, "bytes": 204800 } }, "blocks.5.mlp.mlp.2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 13, "offset": 6758400, "bytes": 2560 } }, "blocks.5.mlp.mlp.2.weight": { "shape": [ 1280, 5120 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 5120, "padded_rows": 1280, "padded_columns": 5120, "data": { "shard": 13, "offset": 6760960, "bytes": 6553600 }, "scales": { "shard": 13, "offset": 13314560, "bytes": 204800 } }, "blocks.5.norm2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 13, "offset": 13519360, "bytes": 2560 } }, "blocks.5.norm2.weight": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 13, "offset": 13521920, "bytes": 2560 } }, "blocks.5.self_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 13, "offset": 13524480, "bytes": 2560 } }, "blocks.5.self_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 13, "offset": 13527040, "bytes": 2560 } }, "blocks.5.self_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 13, "offset": 13529600, "bytes": 2560 } }, "blocks.5.self_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 13, "offset": 13532160, "bytes": 1638400 }, "scales": { "shard": 13, "offset": 15170560, "bytes": 51200 } }, "blocks.5.self_attn.to_qkv.bias": { "shape": [ 3840 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 13, "offset": 15221760, "bytes": 7680 } }, "blocks.5.self_attn.to_qkv.weight": { "shape": [ 3840, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3840, "columns": 1280, "padded_rows": 3840, "padded_columns": 1280, "data": { "shard": 13, "offset": 15229440, "bytes": 4915200 }, "scales": { "shard": 13, "offset": 20144640, "bytes": 153600 } }, "blocks.6.adaLN_modulation.1.bias": { "shape": [ 7680 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 13, "offset": 20298240, "bytes": 15360 } }, "blocks.6.adaLN_modulation.1.weight": { "shape": [ 7680, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 7680, "columns": 1280, "padded_rows": 7680, "padded_columns": 1280, "data": { "shard": 13, "offset": 20313600, "bytes": 9830400 }, "scales": { "shard": 13, "offset": 30144000, "bytes": 307200 } }, "blocks.6.cross_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 13, "offset": 30451200, "bytes": 2560 } }, "blocks.6.cross_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 13, "offset": 30453760, "bytes": 2560 } }, "blocks.6.cross_attn.to_kv.bias": { "shape": [ 2560 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 13, "offset": 30456320, "bytes": 5120 } }, "blocks.6.cross_attn.to_kv.weight": { "shape": [ 2560, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2560, "columns": 768, "padded_rows": 2560, "padded_columns": 768, "data": { "shard": 13, "offset": 30461440, "bytes": 1966080 }, "scales": { "shard": 13, "offset": 32427520, "bytes": 61440 } }, "blocks.6.cross_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 13, "offset": 32488960, "bytes": 2560 } }, "blocks.6.cross_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 13, "offset": 32491520, "bytes": 1638400 }, "scales": { "shard": 13, "offset": 34129920, "bytes": 51200 } }, "blocks.6.cross_attn.to_q.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 13, "offset": 34181120, "bytes": 2560 } }, "blocks.6.cross_attn.to_q.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 13, "offset": 34183680, "bytes": 1638400 }, "scales": { "shard": 13, "offset": 35822080, "bytes": 51200 } }, "blocks.6.mlp.mlp.0.bias": { "shape": [ 5120 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 13, "offset": 35873280, "bytes": 10240 } }, "blocks.6.mlp.mlp.0.weight": { "shape": [ 5120, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 5120, "columns": 1280, "padded_rows": 5120, "padded_columns": 1280, "data": { "shard": 13, "offset": 35883520, "bytes": 6553600 }, "scales": { "shard": 13, "offset": 42437120, "bytes": 204800 } }, "blocks.6.mlp.mlp.2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 13, "offset": 42641920, "bytes": 2560 } }, "blocks.6.mlp.mlp.2.weight": { "shape": [ 1280, 5120 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 5120, "padded_rows": 1280, "padded_columns": 5120, "data": { "shard": 13, "offset": 42644480, "bytes": 6553600 }, "scales": { "shard": 13, "offset": 49198080, "bytes": 204800 } }, "blocks.6.norm2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 13, "offset": 49402880, "bytes": 2560 } }, "blocks.6.norm2.weight": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 13, "offset": 49405440, "bytes": 2560 } }, "blocks.6.self_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 13, "offset": 49408000, "bytes": 2560 } }, "blocks.6.self_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 13, "offset": 49410560, "bytes": 2560 } }, "blocks.6.self_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 13, "offset": 49413120, "bytes": 2560 } }, "blocks.6.self_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 13, "offset": 49415680, "bytes": 1638400 }, "scales": { "shard": 13, "offset": 51054080, "bytes": 51200 } }, "blocks.6.self_attn.to_qkv.bias": { "shape": [ 3840 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 13, "offset": 51105280, "bytes": 7680 } }, "blocks.6.self_attn.to_qkv.weight": { "shape": [ 3840, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3840, "columns": 1280, "padded_rows": 3840, "padded_columns": 1280, "data": { "shard": 13, "offset": 51112960, "bytes": 4915200 }, "scales": { "shard": 13, "offset": 56028160, "bytes": 153600 } }, "blocks.7.adaLN_modulation.1.bias": { "shape": [ 7680 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 13, "offset": 56181760, "bytes": 15360 } }, "blocks.7.adaLN_modulation.1.weight": { "shape": [ 7680, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 7680, "columns": 1280, "padded_rows": 7680, "padded_columns": 1280, "data": { "shard": 13, "offset": 56197120, "bytes": 9830400 }, "scales": { "shard": 13, "offset": 66027520, "bytes": 307200 } }, "blocks.7.cross_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 13, "offset": 66334720, "bytes": 2560 } }, "blocks.7.cross_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 13, "offset": 66337280, "bytes": 2560 } }, "blocks.7.cross_attn.to_kv.bias": { "shape": [ 2560 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 13, "offset": 66339840, "bytes": 5120 } }, "blocks.7.cross_attn.to_kv.weight": { "shape": [ 2560, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2560, "columns": 768, "padded_rows": 2560, "padded_columns": 768, "data": { "shard": 14, "offset": 0, "bytes": 1966080 }, "scales": { "shard": 14, "offset": 1966080, "bytes": 61440 } }, "blocks.7.cross_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 14, "offset": 2027520, "bytes": 2560 } }, "blocks.7.cross_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 14, "offset": 2030080, "bytes": 1638400 }, "scales": { "shard": 14, "offset": 3668480, "bytes": 51200 } }, "blocks.7.cross_attn.to_q.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 14, "offset": 3719680, "bytes": 2560 } }, "blocks.7.cross_attn.to_q.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 14, "offset": 3722240, "bytes": 1638400 }, "scales": { "shard": 14, "offset": 5360640, "bytes": 51200 } }, "blocks.7.mlp.mlp.0.bias": { "shape": [ 5120 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 14, "offset": 5411840, "bytes": 10240 } }, "blocks.7.mlp.mlp.0.weight": { "shape": [ 5120, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 5120, "columns": 1280, "padded_rows": 5120, "padded_columns": 1280, "data": { "shard": 14, "offset": 5422080, "bytes": 6553600 }, "scales": { "shard": 14, "offset": 11975680, "bytes": 204800 } }, "blocks.7.mlp.mlp.2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 14, "offset": 12180480, "bytes": 2560 } }, "blocks.7.mlp.mlp.2.weight": { "shape": [ 1280, 5120 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 5120, "padded_rows": 1280, "padded_columns": 5120, "data": { "shard": 14, "offset": 12183040, "bytes": 6553600 }, "scales": { "shard": 14, "offset": 18736640, "bytes": 204800 } }, "blocks.7.norm2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 14, "offset": 18941440, "bytes": 2560 } }, "blocks.7.norm2.weight": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 14, "offset": 18944000, "bytes": 2560 } }, "blocks.7.self_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 14, "offset": 18946560, "bytes": 2560 } }, "blocks.7.self_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 14, "offset": 18949120, "bytes": 2560 } }, "blocks.7.self_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 14, "offset": 18951680, "bytes": 2560 } }, "blocks.7.self_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 14, "offset": 18954240, "bytes": 1638400 }, "scales": { "shard": 14, "offset": 20592640, "bytes": 51200 } }, "blocks.7.self_attn.to_qkv.bias": { "shape": [ 3840 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 14, "offset": 20643840, "bytes": 7680 } }, "blocks.7.self_attn.to_qkv.weight": { "shape": [ 3840, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3840, "columns": 1280, "padded_rows": 3840, "padded_columns": 1280, "data": { "shard": 14, "offset": 20651520, "bytes": 4915200 }, "scales": { "shard": 14, "offset": 25566720, "bytes": 153600 } }, "blocks.8.adaLN_modulation.1.bias": { "shape": [ 7680 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 14, "offset": 25720320, "bytes": 15360 } }, "blocks.8.adaLN_modulation.1.weight": { "shape": [ 7680, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 7680, "columns": 1280, "padded_rows": 7680, "padded_columns": 1280, "data": { "shard": 14, "offset": 25735680, "bytes": 9830400 }, "scales": { "shard": 14, "offset": 35566080, "bytes": 307200 } }, "blocks.8.cross_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 14, "offset": 35873280, "bytes": 2560 } }, "blocks.8.cross_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 14, "offset": 35875840, "bytes": 2560 } }, "blocks.8.cross_attn.to_kv.bias": { "shape": [ 2560 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 14, "offset": 35878400, "bytes": 5120 } }, "blocks.8.cross_attn.to_kv.weight": { "shape": [ 2560, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2560, "columns": 768, "padded_rows": 2560, "padded_columns": 768, "data": { "shard": 14, "offset": 35883520, "bytes": 1966080 }, "scales": { "shard": 14, "offset": 37849600, "bytes": 61440 } }, "blocks.8.cross_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 14, "offset": 37911040, "bytes": 2560 } }, "blocks.8.cross_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 14, "offset": 37913600, "bytes": 1638400 }, "scales": { "shard": 14, "offset": 39552000, "bytes": 51200 } }, "blocks.8.cross_attn.to_q.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 14, "offset": 39603200, "bytes": 2560 } }, "blocks.8.cross_attn.to_q.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 14, "offset": 39605760, "bytes": 1638400 }, "scales": { "shard": 14, "offset": 41244160, "bytes": 51200 } }, "blocks.8.mlp.mlp.0.bias": { "shape": [ 5120 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 14, "offset": 41295360, "bytes": 10240 } }, "blocks.8.mlp.mlp.0.weight": { "shape": [ 5120, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 5120, "columns": 1280, "padded_rows": 5120, "padded_columns": 1280, "data": { "shard": 14, "offset": 41305600, "bytes": 6553600 }, "scales": { "shard": 14, "offset": 47859200, "bytes": 204800 } }, "blocks.8.mlp.mlp.2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 14, "offset": 48064000, "bytes": 2560 } }, "blocks.8.mlp.mlp.2.weight": { "shape": [ 1280, 5120 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 5120, "padded_rows": 1280, "padded_columns": 5120, "data": { "shard": 14, "offset": 48066560, "bytes": 6553600 }, "scales": { "shard": 14, "offset": 54620160, "bytes": 204800 } }, "blocks.8.norm2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 14, "offset": 54824960, "bytes": 2560 } }, "blocks.8.norm2.weight": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 14, "offset": 54827520, "bytes": 2560 } }, "blocks.8.self_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 14, "offset": 54830080, "bytes": 2560 } }, "blocks.8.self_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 14, "offset": 54832640, "bytes": 2560 } }, "blocks.8.self_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 14, "offset": 54835200, "bytes": 2560 } }, "blocks.8.self_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 14, "offset": 54837760, "bytes": 1638400 }, "scales": { "shard": 14, "offset": 56476160, "bytes": 51200 } }, "blocks.8.self_attn.to_qkv.bias": { "shape": [ 3840 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 14, "offset": 56527360, "bytes": 7680 } }, "blocks.8.self_attn.to_qkv.weight": { "shape": [ 3840, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3840, "columns": 1280, "padded_rows": 3840, "padded_columns": 1280, "data": { "shard": 14, "offset": 56535040, "bytes": 4915200 }, "scales": { "shard": 14, "offset": 61450240, "bytes": 153600 } }, "blocks.9.adaLN_modulation.1.bias": { "shape": [ 7680 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 14, "offset": 61603840, "bytes": 15360 } }, "blocks.9.adaLN_modulation.1.weight": { "shape": [ 7680, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 7680, "columns": 1280, "padded_rows": 7680, "padded_columns": 1280, "data": { "shard": 15, "offset": 0, "bytes": 9830400 }, "scales": { "shard": 15, "offset": 9830400, "bytes": 307200 } }, "blocks.9.cross_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 15, "offset": 10137600, "bytes": 2560 } }, "blocks.9.cross_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 15, "offset": 10140160, "bytes": 2560 } }, "blocks.9.cross_attn.to_kv.bias": { "shape": [ 2560 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 15, "offset": 10142720, "bytes": 5120 } }, "blocks.9.cross_attn.to_kv.weight": { "shape": [ 2560, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2560, "columns": 768, "padded_rows": 2560, "padded_columns": 768, "data": { "shard": 15, "offset": 10147840, "bytes": 1966080 }, "scales": { "shard": 15, "offset": 12113920, "bytes": 61440 } }, "blocks.9.cross_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 15, "offset": 12175360, "bytes": 2560 } }, "blocks.9.cross_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 15, "offset": 12177920, "bytes": 1638400 }, "scales": { "shard": 15, "offset": 13816320, "bytes": 51200 } }, "blocks.9.cross_attn.to_q.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 15, "offset": 13867520, "bytes": 2560 } }, "blocks.9.cross_attn.to_q.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 15, "offset": 13870080, "bytes": 1638400 }, "scales": { "shard": 15, "offset": 15508480, "bytes": 51200 } }, "blocks.9.mlp.mlp.0.bias": { "shape": [ 5120 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 15, "offset": 15559680, "bytes": 10240 } }, "blocks.9.mlp.mlp.0.weight": { "shape": [ 5120, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 5120, "columns": 1280, "padded_rows": 5120, "padded_columns": 1280, "data": { "shard": 15, "offset": 15569920, "bytes": 6553600 }, "scales": { "shard": 15, "offset": 22123520, "bytes": 204800 } }, "blocks.9.mlp.mlp.2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 15, "offset": 22328320, "bytes": 2560 } }, "blocks.9.mlp.mlp.2.weight": { "shape": [ 1280, 5120 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 5120, "padded_rows": 1280, "padded_columns": 5120, "data": { "shard": 15, "offset": 22330880, "bytes": 6553600 }, "scales": { "shard": 15, "offset": 28884480, "bytes": 204800 } }, "blocks.9.norm2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 15, "offset": 29089280, "bytes": 2560 } }, "blocks.9.norm2.weight": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 15, "offset": 29091840, "bytes": 2560 } }, "blocks.9.self_attn.k_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 15, "offset": 29094400, "bytes": 2560 } }, "blocks.9.self_attn.q_rms_norm.gamma": { "shape": [ 16, 80 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 15, "offset": 29096960, "bytes": 2560 } }, "blocks.9.self_attn.to_out.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 15, "offset": 29099520, "bytes": 2560 } }, "blocks.9.self_attn.to_out.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 15, "offset": 29102080, "bytes": 1638400 }, "scales": { "shard": 15, "offset": 30740480, "bytes": 51200 } }, "blocks.9.self_attn.to_qkv.bias": { "shape": [ 3840 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 15, "offset": 30791680, "bytes": 7680 } }, "blocks.9.self_attn.to_qkv.weight": { "shape": [ 3840, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3840, "columns": 1280, "padded_rows": 3840, "padded_columns": 1280, "data": { "shard": 15, "offset": 30799360, "bytes": 4915200 }, "scales": { "shard": 15, "offset": 35714560, "bytes": 153600 } }, "input_layer.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 15, "offset": 35868160, "bytes": 2560 } }, "input_layer.weight": { "shape": [ 1280, 8 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 8, "padded_rows": 1280, "padded_columns": 64, "data": { "shard": 15, "offset": 35870720, "bytes": 81920 }, "scales": { "shard": 15, "offset": 35952640, "bytes": 2560 } }, "out_layer.bias": { "shape": [ 8 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 15, "offset": 35955200, "bytes": 16 } }, "out_layer.weight": { "shape": [ 8, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 8, "columns": 1280, "padded_rows": 16, "padded_columns": 1280, "data": { "shard": 15, "offset": 35955456, "bytes": 20480 }, "scales": { "shard": 15, "offset": 35975936, "bytes": 640 } }, "pos_emb": { "shape": [ 4096, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1280, "padded_rows": 4096, "padded_columns": 1280, "data": { "shard": 15, "offset": 35976704, "bytes": 5242880 }, "scales": { "shard": 15, "offset": 41219584, "bytes": 163840 } }, "t_embedder.mlp.0.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 15, "offset": 41383424, "bytes": 2560 } }, "t_embedder.mlp.0.weight": { "shape": [ 1280, 256 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 256, "padded_rows": 1280, "padded_columns": 256, "data": { "shard": 15, "offset": 41385984, "bytes": 327680 }, "scales": { "shard": 15, "offset": 41713664, "bytes": 10240 } }, "t_embedder.mlp.2.bias": { "shape": [ 1280 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 15, "offset": 41723904, "bytes": 2560 } }, "t_embedder.mlp.2.weight": { "shape": [ 1280, 1280 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1280, "columns": 1280, "padded_rows": 1280, "padded_columns": 1280, "data": { "shard": 15, "offset": 41726464, "bytes": 1638400 }, "scales": { "shard": 15, "offset": 43364864, "bytes": 51200 } } } }