| { |
| "schema": "trellis-webgpu-pack", |
| "version": 1, |
| "stage": "sparseStructureFlow", |
| "upstreamRevision": "e0b00432b8e3a8ecee0df806ab1df9f7281f2be4", |
| "shards": [ |
| { |
| "file": "stage-000.bin", |
| "bytes": 66698240, |
| "sha256": "336b12d08c0783f0c8c49b6ac077854b12b363d7014da5dfa10f8b9d898afcb6" |
| }, |
| { |
| "file": "stage-001.bin", |
| "bytes": 63298560, |
| "sha256": "cf1a86dc354b2cc61cc5615b25ee6b7155ea0383505463b7c1494cd0873fd526" |
| }, |
| { |
| "file": "stage-002.bin", |
| "bytes": 65006080, |
| "sha256": "0163964843d9069d9dd5a4957fc3687f020d1ace80c7dd3a6e2e404c7ed1796a" |
| }, |
| { |
| "file": "stage-003.bin", |
| "bytes": 66344960, |
| "sha256": "e92160701cfa0a8a6a3aed0f306e990b75f87aad874367b8b922d5766b64aa3a" |
| }, |
| { |
| "file": "stage-004.bin", |
| "bytes": 61619200, |
| "sha256": "11d63e67b9571c4921a3f7ebac12f91732c98342ba91acc034a9f852f50495b3" |
| }, |
| { |
| "file": "stage-005.bin", |
| "bytes": 66682880, |
| "sha256": "23b9e6e81b60ab49dd41d5de52c857cbcca4d04eda85e08cb72627ab5fb90bfc" |
| }, |
| { |
| "file": "stage-006.bin", |
| "bytes": 63298560, |
| "sha256": "2373d1cd96c902a88aaff969f7b948e3d2ce8df52fc649ae9e02b2785cc6e535" |
| }, |
| { |
| "file": "stage-007.bin", |
| "bytes": 65006080, |
| "sha256": "750e19abf74ba1145bc6d267573dc9ef8b20cc9d169e2abf0ba3e64a78101e5c" |
| }, |
| { |
| "file": "stage-008.bin", |
| "bytes": 66344960, |
| "sha256": "f2a212769aebc75fabce39b3d1696a663e9b61ca654d1d27bf560622d98dc915" |
| }, |
| { |
| "file": "stage-009.bin", |
| "bytes": 61619200, |
| "sha256": "a5438a63aadcab2cf2f92f4a412a240e43208f4704c564b5407c57c88b3c52bd" |
| }, |
| { |
| "file": "stage-010.bin", |
| "bytes": 66682880, |
| "sha256": "ceb66d586e63b40413741fabcc1c77f802fd235394f9573ca1307a92f61303fa" |
| }, |
| { |
| "file": "stage-011.bin", |
| "bytes": 63298560, |
| "sha256": "adf005f79f4a15741ef02da4509fdb7cfe4b7cb9925a41b5fa0c0d374ff2a77c" |
| }, |
| { |
| "file": "stage-012.bin", |
| "bytes": 65006080, |
| "sha256": "1ae821db1a432de2b0136cf20986f51f014062b2f5f3be5338e26aa1c195fc50" |
| }, |
| { |
| "file": "stage-013.bin", |
| "bytes": 66344960, |
| "sha256": "37b9dcb08d2edb08d990234795c0d7a34b9f1b0180905d6a4b746a49fecac7d1" |
| }, |
| { |
| "file": "stage-014.bin", |
| "bytes": 61619200, |
| "sha256": "e70537d509d17cfff3c6e2bff2e583f4298f716dd7fb2d13a92f0181bcdffdc6" |
| }, |
| { |
| "file": "stage-015.bin", |
| "bytes": 43416064, |
| "sha256": "b7dea07abf9e37e9c01f76d735ef6a44d266dd913124918ccd28a70fdab22442" |
| } |
| ], |
| "tensors": { |
| "blocks.0.adaLN_modulation.1.bias": { |
| "shape": [ |
| 7680 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 0, |
| "bytes": 15360 |
| } |
| }, |
| "blocks.0.adaLN_modulation.1.weight": { |
| "shape": [ |
| 7680, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 7680, |
| "columns": 1280, |
| "padded_rows": 7680, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 0, |
| "offset": 15360, |
| "bytes": 9830400 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 9845760, |
| "bytes": 307200 |
| } |
| }, |
| "blocks.0.cross_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 10152960, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.0.cross_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 10155520, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.0.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2560 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 10158080, |
| "bytes": 5120 |
| } |
| }, |
| "blocks.0.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2560, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2560, |
| "columns": 768, |
| "padded_rows": 2560, |
| "padded_columns": 768, |
| "data": { |
| "shard": 0, |
| "offset": 10163200, |
| "bytes": 1966080 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 12129280, |
| "bytes": 61440 |
| } |
| }, |
| "blocks.0.cross_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 12190720, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.0.cross_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 0, |
| "offset": 12193280, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 13831680, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.0.cross_attn.to_q.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 13882880, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.0.cross_attn.to_q.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 0, |
| "offset": 13885440, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 15523840, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.0.mlp.mlp.0.bias": { |
| "shape": [ |
| 5120 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 15575040, |
| "bytes": 10240 |
| } |
| }, |
| "blocks.0.mlp.mlp.0.weight": { |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 5120, |
| "columns": 1280, |
| "padded_rows": 5120, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 0, |
| "offset": 15585280, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 22138880, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.0.mlp.mlp.2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 22343680, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.0.mlp.mlp.2.weight": { |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 5120, |
| "padded_rows": 1280, |
| "padded_columns": 5120, |
| "data": { |
| "shard": 0, |
| "offset": 22346240, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 28899840, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.0.norm2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 29104640, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.0.norm2.weight": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 29107200, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.0.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 29109760, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.0.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 29112320, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.0.self_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 29114880, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.0.self_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 0, |
| "offset": 29117440, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 30755840, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.0.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3840 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 30807040, |
| "bytes": 7680 |
| } |
| }, |
| "blocks.0.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3840, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3840, |
| "columns": 1280, |
| "padded_rows": 3840, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 0, |
| "offset": 30814720, |
| "bytes": 4915200 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 35729920, |
| "bytes": 153600 |
| } |
| }, |
| "blocks.1.adaLN_modulation.1.bias": { |
| "shape": [ |
| 7680 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 35883520, |
| "bytes": 15360 |
| } |
| }, |
| "blocks.1.adaLN_modulation.1.weight": { |
| "shape": [ |
| 7680, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 7680, |
| "columns": 1280, |
| "padded_rows": 7680, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 0, |
| "offset": 35898880, |
| "bytes": 9830400 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 45729280, |
| "bytes": 307200 |
| } |
| }, |
| "blocks.1.cross_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 46036480, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.1.cross_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 46039040, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.1.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2560 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 46041600, |
| "bytes": 5120 |
| } |
| }, |
| "blocks.1.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2560, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2560, |
| "columns": 768, |
| "padded_rows": 2560, |
| "padded_columns": 768, |
| "data": { |
| "shard": 0, |
| "offset": 46046720, |
| "bytes": 1966080 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 48012800, |
| "bytes": 61440 |
| } |
| }, |
| "blocks.1.cross_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 48074240, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.1.cross_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 0, |
| "offset": 48076800, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 49715200, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.1.cross_attn.to_q.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 49766400, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.1.cross_attn.to_q.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 0, |
| "offset": 49768960, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 51407360, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.1.mlp.mlp.0.bias": { |
| "shape": [ |
| 5120 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 51458560, |
| "bytes": 10240 |
| } |
| }, |
| "blocks.1.mlp.mlp.0.weight": { |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 5120, |
| "columns": 1280, |
| "padded_rows": 5120, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 0, |
| "offset": 51468800, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 58022400, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.1.mlp.mlp.2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 58227200, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.1.mlp.mlp.2.weight": { |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 5120, |
| "padded_rows": 1280, |
| "padded_columns": 5120, |
| "data": { |
| "shard": 0, |
| "offset": 58229760, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 64783360, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.1.norm2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 64988160, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.1.norm2.weight": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 64990720, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.1.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 64993280, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.1.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 64995840, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.1.self_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 64998400, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.1.self_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 0, |
| "offset": 65000960, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 66639360, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.1.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3840 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 66690560, |
| "bytes": 7680 |
| } |
| }, |
| "blocks.1.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3840, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3840, |
| "columns": 1280, |
| "padded_rows": 3840, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 1, |
| "offset": 0, |
| "bytes": 4915200 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 4915200, |
| "bytes": 153600 |
| } |
| }, |
| "blocks.10.adaLN_modulation.1.bias": { |
| "shape": [ |
| 7680 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 5068800, |
| "bytes": 15360 |
| } |
| }, |
| "blocks.10.adaLN_modulation.1.weight": { |
| "shape": [ |
| 7680, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 7680, |
| "columns": 1280, |
| "padded_rows": 7680, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 1, |
| "offset": 5084160, |
| "bytes": 9830400 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 14914560, |
| "bytes": 307200 |
| } |
| }, |
| "blocks.10.cross_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 15221760, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.10.cross_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 15224320, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.10.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2560 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 15226880, |
| "bytes": 5120 |
| } |
| }, |
| "blocks.10.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2560, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2560, |
| "columns": 768, |
| "padded_rows": 2560, |
| "padded_columns": 768, |
| "data": { |
| "shard": 1, |
| "offset": 15232000, |
| "bytes": 1966080 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 17198080, |
| "bytes": 61440 |
| } |
| }, |
| "blocks.10.cross_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 17259520, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.10.cross_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 1, |
| "offset": 17262080, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 18900480, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.10.cross_attn.to_q.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 18951680, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.10.cross_attn.to_q.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 1, |
| "offset": 18954240, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 20592640, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.10.mlp.mlp.0.bias": { |
| "shape": [ |
| 5120 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 20643840, |
| "bytes": 10240 |
| } |
| }, |
| "blocks.10.mlp.mlp.0.weight": { |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 5120, |
| "columns": 1280, |
| "padded_rows": 5120, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 1, |
| "offset": 20654080, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 27207680, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.10.mlp.mlp.2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 27412480, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.10.mlp.mlp.2.weight": { |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 5120, |
| "padded_rows": 1280, |
| "padded_columns": 5120, |
| "data": { |
| "shard": 1, |
| "offset": 27415040, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 33968640, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.10.norm2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 34173440, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.10.norm2.weight": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 34176000, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.10.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 34178560, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.10.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 34181120, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.10.self_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 34183680, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.10.self_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 1, |
| "offset": 34186240, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 35824640, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.10.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3840 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 35875840, |
| "bytes": 7680 |
| } |
| }, |
| "blocks.10.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3840, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3840, |
| "columns": 1280, |
| "padded_rows": 3840, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 1, |
| "offset": 35883520, |
| "bytes": 4915200 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 40798720, |
| "bytes": 153600 |
| } |
| }, |
| "blocks.11.adaLN_modulation.1.bias": { |
| "shape": [ |
| 7680 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 40952320, |
| "bytes": 15360 |
| } |
| }, |
| "blocks.11.adaLN_modulation.1.weight": { |
| "shape": [ |
| 7680, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 7680, |
| "columns": 1280, |
| "padded_rows": 7680, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 1, |
| "offset": 40967680, |
| "bytes": 9830400 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 50798080, |
| "bytes": 307200 |
| } |
| }, |
| "blocks.11.cross_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 51105280, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.11.cross_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 51107840, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.11.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2560 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 51110400, |
| "bytes": 5120 |
| } |
| }, |
| "blocks.11.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2560, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2560, |
| "columns": 768, |
| "padded_rows": 2560, |
| "padded_columns": 768, |
| "data": { |
| "shard": 1, |
| "offset": 51115520, |
| "bytes": 1966080 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 53081600, |
| "bytes": 61440 |
| } |
| }, |
| "blocks.11.cross_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 53143040, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.11.cross_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 1, |
| "offset": 53145600, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 54784000, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.11.cross_attn.to_q.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 54835200, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.11.cross_attn.to_q.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 1, |
| "offset": 54837760, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 56476160, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.11.mlp.mlp.0.bias": { |
| "shape": [ |
| 5120 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 56527360, |
| "bytes": 10240 |
| } |
| }, |
| "blocks.11.mlp.mlp.0.weight": { |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 5120, |
| "columns": 1280, |
| "padded_rows": 5120, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 1, |
| "offset": 56537600, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 63091200, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.11.mlp.mlp.2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 63296000, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.11.mlp.mlp.2.weight": { |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 5120, |
| "padded_rows": 1280, |
| "padded_columns": 5120, |
| "data": { |
| "shard": 2, |
| "offset": 0, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 6553600, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.11.norm2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 6758400, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.11.norm2.weight": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 6760960, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.11.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 6763520, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.11.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 6766080, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.11.self_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 6768640, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.11.self_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 2, |
| "offset": 6771200, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 8409600, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.11.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3840 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 8460800, |
| "bytes": 7680 |
| } |
| }, |
| "blocks.11.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3840, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3840, |
| "columns": 1280, |
| "padded_rows": 3840, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 2, |
| "offset": 8468480, |
| "bytes": 4915200 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 13383680, |
| "bytes": 153600 |
| } |
| }, |
| "blocks.12.adaLN_modulation.1.bias": { |
| "shape": [ |
| 7680 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 13537280, |
| "bytes": 15360 |
| } |
| }, |
| "blocks.12.adaLN_modulation.1.weight": { |
| "shape": [ |
| 7680, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 7680, |
| "columns": 1280, |
| "padded_rows": 7680, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 2, |
| "offset": 13552640, |
| "bytes": 9830400 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 23383040, |
| "bytes": 307200 |
| } |
| }, |
| "blocks.12.cross_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 23690240, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.12.cross_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 23692800, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.12.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2560 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 23695360, |
| "bytes": 5120 |
| } |
| }, |
| "blocks.12.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2560, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2560, |
| "columns": 768, |
| "padded_rows": 2560, |
| "padded_columns": 768, |
| "data": { |
| "shard": 2, |
| "offset": 23700480, |
| "bytes": 1966080 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 25666560, |
| "bytes": 61440 |
| } |
| }, |
| "blocks.12.cross_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 25728000, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.12.cross_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 2, |
| "offset": 25730560, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 27368960, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.12.cross_attn.to_q.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 27420160, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.12.cross_attn.to_q.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 2, |
| "offset": 27422720, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 29061120, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.12.mlp.mlp.0.bias": { |
| "shape": [ |
| 5120 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 29112320, |
| "bytes": 10240 |
| } |
| }, |
| "blocks.12.mlp.mlp.0.weight": { |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 5120, |
| "columns": 1280, |
| "padded_rows": 5120, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 2, |
| "offset": 29122560, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 35676160, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.12.mlp.mlp.2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 35880960, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.12.mlp.mlp.2.weight": { |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 5120, |
| "padded_rows": 1280, |
| "padded_columns": 5120, |
| "data": { |
| "shard": 2, |
| "offset": 35883520, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 42437120, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.12.norm2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 42641920, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.12.norm2.weight": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 42644480, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.12.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 42647040, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.12.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 42649600, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.12.self_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 42652160, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.12.self_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 2, |
| "offset": 42654720, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 44293120, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.12.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3840 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 44344320, |
| "bytes": 7680 |
| } |
| }, |
| "blocks.12.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3840, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3840, |
| "columns": 1280, |
| "padded_rows": 3840, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 2, |
| "offset": 44352000, |
| "bytes": 4915200 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 49267200, |
| "bytes": 153600 |
| } |
| }, |
| "blocks.13.adaLN_modulation.1.bias": { |
| "shape": [ |
| 7680 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 49420800, |
| "bytes": 15360 |
| } |
| }, |
| "blocks.13.adaLN_modulation.1.weight": { |
| "shape": [ |
| 7680, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 7680, |
| "columns": 1280, |
| "padded_rows": 7680, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 2, |
| "offset": 49436160, |
| "bytes": 9830400 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 59266560, |
| "bytes": 307200 |
| } |
| }, |
| "blocks.13.cross_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 59573760, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.13.cross_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 59576320, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.13.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2560 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 59578880, |
| "bytes": 5120 |
| } |
| }, |
| "blocks.13.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2560, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2560, |
| "columns": 768, |
| "padded_rows": 2560, |
| "padded_columns": 768, |
| "data": { |
| "shard": 2, |
| "offset": 59584000, |
| "bytes": 1966080 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 61550080, |
| "bytes": 61440 |
| } |
| }, |
| "blocks.13.cross_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 61611520, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.13.cross_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 2, |
| "offset": 61614080, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 63252480, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.13.cross_attn.to_q.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 63303680, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.13.cross_attn.to_q.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 2, |
| "offset": 63306240, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 64944640, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.13.mlp.mlp.0.bias": { |
| "shape": [ |
| 5120 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 64995840, |
| "bytes": 10240 |
| } |
| }, |
| "blocks.13.mlp.mlp.0.weight": { |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 5120, |
| "columns": 1280, |
| "padded_rows": 5120, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 3, |
| "offset": 0, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 6553600, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.13.mlp.mlp.2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 6758400, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.13.mlp.mlp.2.weight": { |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 5120, |
| "padded_rows": 1280, |
| "padded_columns": 5120, |
| "data": { |
| "shard": 3, |
| "offset": 6760960, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 13314560, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.13.norm2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 13519360, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.13.norm2.weight": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 13521920, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.13.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 13524480, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.13.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 13527040, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.13.self_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 13529600, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.13.self_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 3, |
| "offset": 13532160, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 15170560, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.13.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3840 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 15221760, |
| "bytes": 7680 |
| } |
| }, |
| "blocks.13.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3840, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3840, |
| "columns": 1280, |
| "padded_rows": 3840, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 3, |
| "offset": 15229440, |
| "bytes": 4915200 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 20144640, |
| "bytes": 153600 |
| } |
| }, |
| "blocks.14.adaLN_modulation.1.bias": { |
| "shape": [ |
| 7680 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 20298240, |
| "bytes": 15360 |
| } |
| }, |
| "blocks.14.adaLN_modulation.1.weight": { |
| "shape": [ |
| 7680, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 7680, |
| "columns": 1280, |
| "padded_rows": 7680, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 3, |
| "offset": 20313600, |
| "bytes": 9830400 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 30144000, |
| "bytes": 307200 |
| } |
| }, |
| "blocks.14.cross_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 30451200, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.14.cross_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 30453760, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.14.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2560 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 30456320, |
| "bytes": 5120 |
| } |
| }, |
| "blocks.14.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2560, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2560, |
| "columns": 768, |
| "padded_rows": 2560, |
| "padded_columns": 768, |
| "data": { |
| "shard": 3, |
| "offset": 30461440, |
| "bytes": 1966080 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 32427520, |
| "bytes": 61440 |
| } |
| }, |
| "blocks.14.cross_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 32488960, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.14.cross_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 3, |
| "offset": 32491520, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 34129920, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.14.cross_attn.to_q.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 34181120, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.14.cross_attn.to_q.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 3, |
| "offset": 34183680, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 35822080, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.14.mlp.mlp.0.bias": { |
| "shape": [ |
| 5120 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 35873280, |
| "bytes": 10240 |
| } |
| }, |
| "blocks.14.mlp.mlp.0.weight": { |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 5120, |
| "columns": 1280, |
| "padded_rows": 5120, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 3, |
| "offset": 35883520, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 42437120, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.14.mlp.mlp.2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 42641920, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.14.mlp.mlp.2.weight": { |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 5120, |
| "padded_rows": 1280, |
| "padded_columns": 5120, |
| "data": { |
| "shard": 3, |
| "offset": 42644480, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 49198080, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.14.norm2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 49402880, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.14.norm2.weight": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 49405440, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.14.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 49408000, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.14.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 49410560, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.14.self_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 49413120, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.14.self_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 3, |
| "offset": 49415680, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 51054080, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.14.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3840 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 51105280, |
| "bytes": 7680 |
| } |
| }, |
| "blocks.14.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3840, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3840, |
| "columns": 1280, |
| "padded_rows": 3840, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 3, |
| "offset": 51112960, |
| "bytes": 4915200 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 56028160, |
| "bytes": 153600 |
| } |
| }, |
| "blocks.15.adaLN_modulation.1.bias": { |
| "shape": [ |
| 7680 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 56181760, |
| "bytes": 15360 |
| } |
| }, |
| "blocks.15.adaLN_modulation.1.weight": { |
| "shape": [ |
| 7680, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 7680, |
| "columns": 1280, |
| "padded_rows": 7680, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 3, |
| "offset": 56197120, |
| "bytes": 9830400 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 66027520, |
| "bytes": 307200 |
| } |
| }, |
| "blocks.15.cross_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 66334720, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.15.cross_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 66337280, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.15.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2560 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 66339840, |
| "bytes": 5120 |
| } |
| }, |
| "blocks.15.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2560, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2560, |
| "columns": 768, |
| "padded_rows": 2560, |
| "padded_columns": 768, |
| "data": { |
| "shard": 4, |
| "offset": 0, |
| "bytes": 1966080 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 1966080, |
| "bytes": 61440 |
| } |
| }, |
| "blocks.15.cross_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 2027520, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.15.cross_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 4, |
| "offset": 2030080, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 3668480, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.15.cross_attn.to_q.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 3719680, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.15.cross_attn.to_q.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 4, |
| "offset": 3722240, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 5360640, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.15.mlp.mlp.0.bias": { |
| "shape": [ |
| 5120 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 5411840, |
| "bytes": 10240 |
| } |
| }, |
| "blocks.15.mlp.mlp.0.weight": { |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 5120, |
| "columns": 1280, |
| "padded_rows": 5120, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 4, |
| "offset": 5422080, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 11975680, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.15.mlp.mlp.2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 12180480, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.15.mlp.mlp.2.weight": { |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 5120, |
| "padded_rows": 1280, |
| "padded_columns": 5120, |
| "data": { |
| "shard": 4, |
| "offset": 12183040, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 18736640, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.15.norm2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 18941440, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.15.norm2.weight": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 18944000, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.15.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 18946560, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.15.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 18949120, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.15.self_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 18951680, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.15.self_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 4, |
| "offset": 18954240, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 20592640, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.15.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3840 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 20643840, |
| "bytes": 7680 |
| } |
| }, |
| "blocks.15.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3840, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3840, |
| "columns": 1280, |
| "padded_rows": 3840, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 4, |
| "offset": 20651520, |
| "bytes": 4915200 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 25566720, |
| "bytes": 153600 |
| } |
| }, |
| "blocks.16.adaLN_modulation.1.bias": { |
| "shape": [ |
| 7680 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 25720320, |
| "bytes": 15360 |
| } |
| }, |
| "blocks.16.adaLN_modulation.1.weight": { |
| "shape": [ |
| 7680, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 7680, |
| "columns": 1280, |
| "padded_rows": 7680, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 4, |
| "offset": 25735680, |
| "bytes": 9830400 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 35566080, |
| "bytes": 307200 |
| } |
| }, |
| "blocks.16.cross_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 35873280, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.16.cross_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 35875840, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.16.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2560 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 35878400, |
| "bytes": 5120 |
| } |
| }, |
| "blocks.16.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2560, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2560, |
| "columns": 768, |
| "padded_rows": 2560, |
| "padded_columns": 768, |
| "data": { |
| "shard": 4, |
| "offset": 35883520, |
| "bytes": 1966080 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 37849600, |
| "bytes": 61440 |
| } |
| }, |
| "blocks.16.cross_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 37911040, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.16.cross_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 4, |
| "offset": 37913600, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 39552000, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.16.cross_attn.to_q.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 39603200, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.16.cross_attn.to_q.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 4, |
| "offset": 39605760, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 41244160, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.16.mlp.mlp.0.bias": { |
| "shape": [ |
| 5120 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 41295360, |
| "bytes": 10240 |
| } |
| }, |
| "blocks.16.mlp.mlp.0.weight": { |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 5120, |
| "columns": 1280, |
| "padded_rows": 5120, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 4, |
| "offset": 41305600, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 47859200, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.16.mlp.mlp.2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 48064000, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.16.mlp.mlp.2.weight": { |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 5120, |
| "padded_rows": 1280, |
| "padded_columns": 5120, |
| "data": { |
| "shard": 4, |
| "offset": 48066560, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 54620160, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.16.norm2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 54824960, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.16.norm2.weight": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 54827520, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.16.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 54830080, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.16.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 54832640, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.16.self_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 54835200, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.16.self_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 4, |
| "offset": 54837760, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 56476160, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.16.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3840 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 56527360, |
| "bytes": 7680 |
| } |
| }, |
| "blocks.16.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3840, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3840, |
| "columns": 1280, |
| "padded_rows": 3840, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 4, |
| "offset": 56535040, |
| "bytes": 4915200 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 61450240, |
| "bytes": 153600 |
| } |
| }, |
| "blocks.17.adaLN_modulation.1.bias": { |
| "shape": [ |
| 7680 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 61603840, |
| "bytes": 15360 |
| } |
| }, |
| "blocks.17.adaLN_modulation.1.weight": { |
| "shape": [ |
| 7680, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 7680, |
| "columns": 1280, |
| "padded_rows": 7680, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 5, |
| "offset": 0, |
| "bytes": 9830400 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 9830400, |
| "bytes": 307200 |
| } |
| }, |
| "blocks.17.cross_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 10137600, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.17.cross_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 10140160, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.17.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2560 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 10142720, |
| "bytes": 5120 |
| } |
| }, |
| "blocks.17.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2560, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2560, |
| "columns": 768, |
| "padded_rows": 2560, |
| "padded_columns": 768, |
| "data": { |
| "shard": 5, |
| "offset": 10147840, |
| "bytes": 1966080 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 12113920, |
| "bytes": 61440 |
| } |
| }, |
| "blocks.17.cross_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 12175360, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.17.cross_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 5, |
| "offset": 12177920, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 13816320, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.17.cross_attn.to_q.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 13867520, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.17.cross_attn.to_q.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 5, |
| "offset": 13870080, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 15508480, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.17.mlp.mlp.0.bias": { |
| "shape": [ |
| 5120 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 15559680, |
| "bytes": 10240 |
| } |
| }, |
| "blocks.17.mlp.mlp.0.weight": { |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 5120, |
| "columns": 1280, |
| "padded_rows": 5120, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 5, |
| "offset": 15569920, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 22123520, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.17.mlp.mlp.2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 22328320, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.17.mlp.mlp.2.weight": { |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 5120, |
| "padded_rows": 1280, |
| "padded_columns": 5120, |
| "data": { |
| "shard": 5, |
| "offset": 22330880, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 28884480, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.17.norm2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 29089280, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.17.norm2.weight": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 29091840, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.17.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 29094400, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.17.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 29096960, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.17.self_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 29099520, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.17.self_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 5, |
| "offset": 29102080, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 30740480, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.17.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3840 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 30791680, |
| "bytes": 7680 |
| } |
| }, |
| "blocks.17.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3840, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3840, |
| "columns": 1280, |
| "padded_rows": 3840, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 5, |
| "offset": 30799360, |
| "bytes": 4915200 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 35714560, |
| "bytes": 153600 |
| } |
| }, |
| "blocks.18.adaLN_modulation.1.bias": { |
| "shape": [ |
| 7680 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 35868160, |
| "bytes": 15360 |
| } |
| }, |
| "blocks.18.adaLN_modulation.1.weight": { |
| "shape": [ |
| 7680, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 7680, |
| "columns": 1280, |
| "padded_rows": 7680, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 5, |
| "offset": 35883520, |
| "bytes": 9830400 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 45713920, |
| "bytes": 307200 |
| } |
| }, |
| "blocks.18.cross_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 46021120, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.18.cross_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 46023680, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.18.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2560 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 46026240, |
| "bytes": 5120 |
| } |
| }, |
| "blocks.18.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2560, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2560, |
| "columns": 768, |
| "padded_rows": 2560, |
| "padded_columns": 768, |
| "data": { |
| "shard": 5, |
| "offset": 46031360, |
| "bytes": 1966080 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 47997440, |
| "bytes": 61440 |
| } |
| }, |
| "blocks.18.cross_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 48058880, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.18.cross_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 5, |
| "offset": 48061440, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 49699840, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.18.cross_attn.to_q.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 49751040, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.18.cross_attn.to_q.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 5, |
| "offset": 49753600, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 51392000, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.18.mlp.mlp.0.bias": { |
| "shape": [ |
| 5120 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 51443200, |
| "bytes": 10240 |
| } |
| }, |
| "blocks.18.mlp.mlp.0.weight": { |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 5120, |
| "columns": 1280, |
| "padded_rows": 5120, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 5, |
| "offset": 51453440, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 58007040, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.18.mlp.mlp.2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 58211840, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.18.mlp.mlp.2.weight": { |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 5120, |
| "padded_rows": 1280, |
| "padded_columns": 5120, |
| "data": { |
| "shard": 5, |
| "offset": 58214400, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 64768000, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.18.norm2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 64972800, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.18.norm2.weight": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 64975360, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.18.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 64977920, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.18.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 64980480, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.18.self_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 64983040, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.18.self_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 5, |
| "offset": 64985600, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 66624000, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.18.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3840 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 66675200, |
| "bytes": 7680 |
| } |
| }, |
| "blocks.18.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3840, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3840, |
| "columns": 1280, |
| "padded_rows": 3840, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 6, |
| "offset": 0, |
| "bytes": 4915200 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 4915200, |
| "bytes": 153600 |
| } |
| }, |
| "blocks.19.adaLN_modulation.1.bias": { |
| "shape": [ |
| 7680 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 5068800, |
| "bytes": 15360 |
| } |
| }, |
| "blocks.19.adaLN_modulation.1.weight": { |
| "shape": [ |
| 7680, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 7680, |
| "columns": 1280, |
| "padded_rows": 7680, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 6, |
| "offset": 5084160, |
| "bytes": 9830400 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 14914560, |
| "bytes": 307200 |
| } |
| }, |
| "blocks.19.cross_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 15221760, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.19.cross_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 15224320, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.19.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2560 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 15226880, |
| "bytes": 5120 |
| } |
| }, |
| "blocks.19.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2560, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2560, |
| "columns": 768, |
| "padded_rows": 2560, |
| "padded_columns": 768, |
| "data": { |
| "shard": 6, |
| "offset": 15232000, |
| "bytes": 1966080 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 17198080, |
| "bytes": 61440 |
| } |
| }, |
| "blocks.19.cross_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 17259520, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.19.cross_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 6, |
| "offset": 17262080, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 18900480, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.19.cross_attn.to_q.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 18951680, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.19.cross_attn.to_q.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 6, |
| "offset": 18954240, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 20592640, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.19.mlp.mlp.0.bias": { |
| "shape": [ |
| 5120 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 20643840, |
| "bytes": 10240 |
| } |
| }, |
| "blocks.19.mlp.mlp.0.weight": { |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 5120, |
| "columns": 1280, |
| "padded_rows": 5120, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 6, |
| "offset": 20654080, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 27207680, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.19.mlp.mlp.2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 27412480, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.19.mlp.mlp.2.weight": { |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 5120, |
| "padded_rows": 1280, |
| "padded_columns": 5120, |
| "data": { |
| "shard": 6, |
| "offset": 27415040, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 33968640, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.19.norm2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 34173440, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.19.norm2.weight": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 34176000, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.19.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 34178560, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.19.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 34181120, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.19.self_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 34183680, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.19.self_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 6, |
| "offset": 34186240, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 35824640, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.19.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3840 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 35875840, |
| "bytes": 7680 |
| } |
| }, |
| "blocks.19.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3840, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3840, |
| "columns": 1280, |
| "padded_rows": 3840, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 6, |
| "offset": 35883520, |
| "bytes": 4915200 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 40798720, |
| "bytes": 153600 |
| } |
| }, |
| "blocks.2.adaLN_modulation.1.bias": { |
| "shape": [ |
| 7680 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 40952320, |
| "bytes": 15360 |
| } |
| }, |
| "blocks.2.adaLN_modulation.1.weight": { |
| "shape": [ |
| 7680, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 7680, |
| "columns": 1280, |
| "padded_rows": 7680, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 6, |
| "offset": 40967680, |
| "bytes": 9830400 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 50798080, |
| "bytes": 307200 |
| } |
| }, |
| "blocks.2.cross_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 51105280, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.2.cross_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 51107840, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.2.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2560 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 51110400, |
| "bytes": 5120 |
| } |
| }, |
| "blocks.2.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2560, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2560, |
| "columns": 768, |
| "padded_rows": 2560, |
| "padded_columns": 768, |
| "data": { |
| "shard": 6, |
| "offset": 51115520, |
| "bytes": 1966080 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 53081600, |
| "bytes": 61440 |
| } |
| }, |
| "blocks.2.cross_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 53143040, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.2.cross_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 6, |
| "offset": 53145600, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 54784000, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.2.cross_attn.to_q.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 54835200, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.2.cross_attn.to_q.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 6, |
| "offset": 54837760, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 56476160, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.2.mlp.mlp.0.bias": { |
| "shape": [ |
| 5120 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 56527360, |
| "bytes": 10240 |
| } |
| }, |
| "blocks.2.mlp.mlp.0.weight": { |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 5120, |
| "columns": 1280, |
| "padded_rows": 5120, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 6, |
| "offset": 56537600, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 63091200, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.2.mlp.mlp.2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 63296000, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.2.mlp.mlp.2.weight": { |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 5120, |
| "padded_rows": 1280, |
| "padded_columns": 5120, |
| "data": { |
| "shard": 7, |
| "offset": 0, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 6553600, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.2.norm2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 6758400, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.2.norm2.weight": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 6760960, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.2.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 6763520, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.2.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 6766080, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.2.self_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 6768640, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.2.self_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 7, |
| "offset": 6771200, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 8409600, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.2.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3840 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 8460800, |
| "bytes": 7680 |
| } |
| }, |
| "blocks.2.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3840, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3840, |
| "columns": 1280, |
| "padded_rows": 3840, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 7, |
| "offset": 8468480, |
| "bytes": 4915200 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 13383680, |
| "bytes": 153600 |
| } |
| }, |
| "blocks.20.adaLN_modulation.1.bias": { |
| "shape": [ |
| 7680 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 13537280, |
| "bytes": 15360 |
| } |
| }, |
| "blocks.20.adaLN_modulation.1.weight": { |
| "shape": [ |
| 7680, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 7680, |
| "columns": 1280, |
| "padded_rows": 7680, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 7, |
| "offset": 13552640, |
| "bytes": 9830400 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 23383040, |
| "bytes": 307200 |
| } |
| }, |
| "blocks.20.cross_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 23690240, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.20.cross_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 23692800, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.20.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2560 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 23695360, |
| "bytes": 5120 |
| } |
| }, |
| "blocks.20.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2560, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2560, |
| "columns": 768, |
| "padded_rows": 2560, |
| "padded_columns": 768, |
| "data": { |
| "shard": 7, |
| "offset": 23700480, |
| "bytes": 1966080 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 25666560, |
| "bytes": 61440 |
| } |
| }, |
| "blocks.20.cross_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 25728000, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.20.cross_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 7, |
| "offset": 25730560, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 27368960, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.20.cross_attn.to_q.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 27420160, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.20.cross_attn.to_q.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 7, |
| "offset": 27422720, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 29061120, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.20.mlp.mlp.0.bias": { |
| "shape": [ |
| 5120 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 29112320, |
| "bytes": 10240 |
| } |
| }, |
| "blocks.20.mlp.mlp.0.weight": { |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 5120, |
| "columns": 1280, |
| "padded_rows": 5120, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 7, |
| "offset": 29122560, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 35676160, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.20.mlp.mlp.2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 35880960, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.20.mlp.mlp.2.weight": { |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 5120, |
| "padded_rows": 1280, |
| "padded_columns": 5120, |
| "data": { |
| "shard": 7, |
| "offset": 35883520, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 42437120, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.20.norm2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 42641920, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.20.norm2.weight": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 42644480, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.20.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 42647040, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.20.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 42649600, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.20.self_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 42652160, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.20.self_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 7, |
| "offset": 42654720, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 44293120, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.20.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3840 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 44344320, |
| "bytes": 7680 |
| } |
| }, |
| "blocks.20.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3840, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3840, |
| "columns": 1280, |
| "padded_rows": 3840, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 7, |
| "offset": 44352000, |
| "bytes": 4915200 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 49267200, |
| "bytes": 153600 |
| } |
| }, |
| "blocks.21.adaLN_modulation.1.bias": { |
| "shape": [ |
| 7680 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 49420800, |
| "bytes": 15360 |
| } |
| }, |
| "blocks.21.adaLN_modulation.1.weight": { |
| "shape": [ |
| 7680, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 7680, |
| "columns": 1280, |
| "padded_rows": 7680, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 7, |
| "offset": 49436160, |
| "bytes": 9830400 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 59266560, |
| "bytes": 307200 |
| } |
| }, |
| "blocks.21.cross_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 59573760, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.21.cross_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 59576320, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.21.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2560 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 59578880, |
| "bytes": 5120 |
| } |
| }, |
| "blocks.21.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2560, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2560, |
| "columns": 768, |
| "padded_rows": 2560, |
| "padded_columns": 768, |
| "data": { |
| "shard": 7, |
| "offset": 59584000, |
| "bytes": 1966080 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 61550080, |
| "bytes": 61440 |
| } |
| }, |
| "blocks.21.cross_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 61611520, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.21.cross_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 7, |
| "offset": 61614080, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 63252480, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.21.cross_attn.to_q.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 63303680, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.21.cross_attn.to_q.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 7, |
| "offset": 63306240, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 64944640, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.21.mlp.mlp.0.bias": { |
| "shape": [ |
| 5120 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 64995840, |
| "bytes": 10240 |
| } |
| }, |
| "blocks.21.mlp.mlp.0.weight": { |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 5120, |
| "columns": 1280, |
| "padded_rows": 5120, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 8, |
| "offset": 0, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 8, |
| "offset": 6553600, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.21.mlp.mlp.2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 6758400, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.21.mlp.mlp.2.weight": { |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 5120, |
| "padded_rows": 1280, |
| "padded_columns": 5120, |
| "data": { |
| "shard": 8, |
| "offset": 6760960, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 8, |
| "offset": 13314560, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.21.norm2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 13519360, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.21.norm2.weight": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 13521920, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.21.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 13524480, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.21.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 13527040, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.21.self_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 13529600, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.21.self_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 8, |
| "offset": 13532160, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 8, |
| "offset": 15170560, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.21.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3840 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 15221760, |
| "bytes": 7680 |
| } |
| }, |
| "blocks.21.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3840, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3840, |
| "columns": 1280, |
| "padded_rows": 3840, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 8, |
| "offset": 15229440, |
| "bytes": 4915200 |
| }, |
| "scales": { |
| "shard": 8, |
| "offset": 20144640, |
| "bytes": 153600 |
| } |
| }, |
| "blocks.22.adaLN_modulation.1.bias": { |
| "shape": [ |
| 7680 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 20298240, |
| "bytes": 15360 |
| } |
| }, |
| "blocks.22.adaLN_modulation.1.weight": { |
| "shape": [ |
| 7680, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 7680, |
| "columns": 1280, |
| "padded_rows": 7680, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 8, |
| "offset": 20313600, |
| "bytes": 9830400 |
| }, |
| "scales": { |
| "shard": 8, |
| "offset": 30144000, |
| "bytes": 307200 |
| } |
| }, |
| "blocks.22.cross_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 30451200, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.22.cross_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 30453760, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.22.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2560 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 30456320, |
| "bytes": 5120 |
| } |
| }, |
| "blocks.22.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2560, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2560, |
| "columns": 768, |
| "padded_rows": 2560, |
| "padded_columns": 768, |
| "data": { |
| "shard": 8, |
| "offset": 30461440, |
| "bytes": 1966080 |
| }, |
| "scales": { |
| "shard": 8, |
| "offset": 32427520, |
| "bytes": 61440 |
| } |
| }, |
| "blocks.22.cross_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 32488960, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.22.cross_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 8, |
| "offset": 32491520, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 8, |
| "offset": 34129920, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.22.cross_attn.to_q.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 34181120, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.22.cross_attn.to_q.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 8, |
| "offset": 34183680, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 8, |
| "offset": 35822080, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.22.mlp.mlp.0.bias": { |
| "shape": [ |
| 5120 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 35873280, |
| "bytes": 10240 |
| } |
| }, |
| "blocks.22.mlp.mlp.0.weight": { |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 5120, |
| "columns": 1280, |
| "padded_rows": 5120, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 8, |
| "offset": 35883520, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 8, |
| "offset": 42437120, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.22.mlp.mlp.2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 42641920, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.22.mlp.mlp.2.weight": { |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 5120, |
| "padded_rows": 1280, |
| "padded_columns": 5120, |
| "data": { |
| "shard": 8, |
| "offset": 42644480, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 8, |
| "offset": 49198080, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.22.norm2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 49402880, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.22.norm2.weight": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 49405440, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.22.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 49408000, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.22.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 49410560, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.22.self_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 49413120, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.22.self_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 8, |
| "offset": 49415680, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 8, |
| "offset": 51054080, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.22.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3840 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 51105280, |
| "bytes": 7680 |
| } |
| }, |
| "blocks.22.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3840, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3840, |
| "columns": 1280, |
| "padded_rows": 3840, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 8, |
| "offset": 51112960, |
| "bytes": 4915200 |
| }, |
| "scales": { |
| "shard": 8, |
| "offset": 56028160, |
| "bytes": 153600 |
| } |
| }, |
| "blocks.23.adaLN_modulation.1.bias": { |
| "shape": [ |
| 7680 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 56181760, |
| "bytes": 15360 |
| } |
| }, |
| "blocks.23.adaLN_modulation.1.weight": { |
| "shape": [ |
| 7680, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 7680, |
| "columns": 1280, |
| "padded_rows": 7680, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 8, |
| "offset": 56197120, |
| "bytes": 9830400 |
| }, |
| "scales": { |
| "shard": 8, |
| "offset": 66027520, |
| "bytes": 307200 |
| } |
| }, |
| "blocks.23.cross_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 66334720, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.23.cross_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 66337280, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.23.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2560 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 66339840, |
| "bytes": 5120 |
| } |
| }, |
| "blocks.23.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2560, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2560, |
| "columns": 768, |
| "padded_rows": 2560, |
| "padded_columns": 768, |
| "data": { |
| "shard": 9, |
| "offset": 0, |
| "bytes": 1966080 |
| }, |
| "scales": { |
| "shard": 9, |
| "offset": 1966080, |
| "bytes": 61440 |
| } |
| }, |
| "blocks.23.cross_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 2027520, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.23.cross_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 9, |
| "offset": 2030080, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 9, |
| "offset": 3668480, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.23.cross_attn.to_q.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 3719680, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.23.cross_attn.to_q.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 9, |
| "offset": 3722240, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 9, |
| "offset": 5360640, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.23.mlp.mlp.0.bias": { |
| "shape": [ |
| 5120 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 5411840, |
| "bytes": 10240 |
| } |
| }, |
| "blocks.23.mlp.mlp.0.weight": { |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 5120, |
| "columns": 1280, |
| "padded_rows": 5120, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 9, |
| "offset": 5422080, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 9, |
| "offset": 11975680, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.23.mlp.mlp.2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 12180480, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.23.mlp.mlp.2.weight": { |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 5120, |
| "padded_rows": 1280, |
| "padded_columns": 5120, |
| "data": { |
| "shard": 9, |
| "offset": 12183040, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 9, |
| "offset": 18736640, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.23.norm2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 18941440, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.23.norm2.weight": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 18944000, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.23.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 18946560, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.23.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 18949120, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.23.self_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 18951680, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.23.self_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 9, |
| "offset": 18954240, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 9, |
| "offset": 20592640, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.23.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3840 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 20643840, |
| "bytes": 7680 |
| } |
| }, |
| "blocks.23.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3840, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3840, |
| "columns": 1280, |
| "padded_rows": 3840, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 9, |
| "offset": 20651520, |
| "bytes": 4915200 |
| }, |
| "scales": { |
| "shard": 9, |
| "offset": 25566720, |
| "bytes": 153600 |
| } |
| }, |
| "blocks.24.adaLN_modulation.1.bias": { |
| "shape": [ |
| 7680 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 25720320, |
| "bytes": 15360 |
| } |
| }, |
| "blocks.24.adaLN_modulation.1.weight": { |
| "shape": [ |
| 7680, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 7680, |
| "columns": 1280, |
| "padded_rows": 7680, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 9, |
| "offset": 25735680, |
| "bytes": 9830400 |
| }, |
| "scales": { |
| "shard": 9, |
| "offset": 35566080, |
| "bytes": 307200 |
| } |
| }, |
| "blocks.24.cross_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 35873280, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.24.cross_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 35875840, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.24.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2560 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 35878400, |
| "bytes": 5120 |
| } |
| }, |
| "blocks.24.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2560, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2560, |
| "columns": 768, |
| "padded_rows": 2560, |
| "padded_columns": 768, |
| "data": { |
| "shard": 9, |
| "offset": 35883520, |
| "bytes": 1966080 |
| }, |
| "scales": { |
| "shard": 9, |
| "offset": 37849600, |
| "bytes": 61440 |
| } |
| }, |
| "blocks.24.cross_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 37911040, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.24.cross_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 9, |
| "offset": 37913600, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 9, |
| "offset": 39552000, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.24.cross_attn.to_q.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 39603200, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.24.cross_attn.to_q.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 9, |
| "offset": 39605760, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 9, |
| "offset": 41244160, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.24.mlp.mlp.0.bias": { |
| "shape": [ |
| 5120 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 41295360, |
| "bytes": 10240 |
| } |
| }, |
| "blocks.24.mlp.mlp.0.weight": { |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 5120, |
| "columns": 1280, |
| "padded_rows": 5120, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 9, |
| "offset": 41305600, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 9, |
| "offset": 47859200, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.24.mlp.mlp.2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 48064000, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.24.mlp.mlp.2.weight": { |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 5120, |
| "padded_rows": 1280, |
| "padded_columns": 5120, |
| "data": { |
| "shard": 9, |
| "offset": 48066560, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 9, |
| "offset": 54620160, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.24.norm2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 54824960, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.24.norm2.weight": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 54827520, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.24.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 54830080, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.24.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 54832640, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.24.self_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 54835200, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.24.self_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 9, |
| "offset": 54837760, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 9, |
| "offset": 56476160, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.24.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3840 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 56527360, |
| "bytes": 7680 |
| } |
| }, |
| "blocks.24.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3840, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3840, |
| "columns": 1280, |
| "padded_rows": 3840, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 9, |
| "offset": 56535040, |
| "bytes": 4915200 |
| }, |
| "scales": { |
| "shard": 9, |
| "offset": 61450240, |
| "bytes": 153600 |
| } |
| }, |
| "blocks.25.adaLN_modulation.1.bias": { |
| "shape": [ |
| 7680 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 9, |
| "offset": 61603840, |
| "bytes": 15360 |
| } |
| }, |
| "blocks.25.adaLN_modulation.1.weight": { |
| "shape": [ |
| 7680, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 7680, |
| "columns": 1280, |
| "padded_rows": 7680, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 10, |
| "offset": 0, |
| "bytes": 9830400 |
| }, |
| "scales": { |
| "shard": 10, |
| "offset": 9830400, |
| "bytes": 307200 |
| } |
| }, |
| "blocks.25.cross_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 10137600, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.25.cross_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 10140160, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.25.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2560 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 10142720, |
| "bytes": 5120 |
| } |
| }, |
| "blocks.25.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2560, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2560, |
| "columns": 768, |
| "padded_rows": 2560, |
| "padded_columns": 768, |
| "data": { |
| "shard": 10, |
| "offset": 10147840, |
| "bytes": 1966080 |
| }, |
| "scales": { |
| "shard": 10, |
| "offset": 12113920, |
| "bytes": 61440 |
| } |
| }, |
| "blocks.25.cross_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 12175360, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.25.cross_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 10, |
| "offset": 12177920, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 10, |
| "offset": 13816320, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.25.cross_attn.to_q.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 13867520, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.25.cross_attn.to_q.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 10, |
| "offset": 13870080, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 10, |
| "offset": 15508480, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.25.mlp.mlp.0.bias": { |
| "shape": [ |
| 5120 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 15559680, |
| "bytes": 10240 |
| } |
| }, |
| "blocks.25.mlp.mlp.0.weight": { |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 5120, |
| "columns": 1280, |
| "padded_rows": 5120, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 10, |
| "offset": 15569920, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 10, |
| "offset": 22123520, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.25.mlp.mlp.2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 22328320, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.25.mlp.mlp.2.weight": { |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 5120, |
| "padded_rows": 1280, |
| "padded_columns": 5120, |
| "data": { |
| "shard": 10, |
| "offset": 22330880, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 10, |
| "offset": 28884480, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.25.norm2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 29089280, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.25.norm2.weight": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 29091840, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.25.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 29094400, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.25.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 29096960, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.25.self_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 29099520, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.25.self_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 10, |
| "offset": 29102080, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 10, |
| "offset": 30740480, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.25.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3840 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 30791680, |
| "bytes": 7680 |
| } |
| }, |
| "blocks.25.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3840, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3840, |
| "columns": 1280, |
| "padded_rows": 3840, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 10, |
| "offset": 30799360, |
| "bytes": 4915200 |
| }, |
| "scales": { |
| "shard": 10, |
| "offset": 35714560, |
| "bytes": 153600 |
| } |
| }, |
| "blocks.26.adaLN_modulation.1.bias": { |
| "shape": [ |
| 7680 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 35868160, |
| "bytes": 15360 |
| } |
| }, |
| "blocks.26.adaLN_modulation.1.weight": { |
| "shape": [ |
| 7680, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 7680, |
| "columns": 1280, |
| "padded_rows": 7680, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 10, |
| "offset": 35883520, |
| "bytes": 9830400 |
| }, |
| "scales": { |
| "shard": 10, |
| "offset": 45713920, |
| "bytes": 307200 |
| } |
| }, |
| "blocks.26.cross_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 46021120, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.26.cross_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 46023680, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.26.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2560 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 46026240, |
| "bytes": 5120 |
| } |
| }, |
| "blocks.26.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2560, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2560, |
| "columns": 768, |
| "padded_rows": 2560, |
| "padded_columns": 768, |
| "data": { |
| "shard": 10, |
| "offset": 46031360, |
| "bytes": 1966080 |
| }, |
| "scales": { |
| "shard": 10, |
| "offset": 47997440, |
| "bytes": 61440 |
| } |
| }, |
| "blocks.26.cross_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 48058880, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.26.cross_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 10, |
| "offset": 48061440, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 10, |
| "offset": 49699840, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.26.cross_attn.to_q.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 49751040, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.26.cross_attn.to_q.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 10, |
| "offset": 49753600, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 10, |
| "offset": 51392000, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.26.mlp.mlp.0.bias": { |
| "shape": [ |
| 5120 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 51443200, |
| "bytes": 10240 |
| } |
| }, |
| "blocks.26.mlp.mlp.0.weight": { |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 5120, |
| "columns": 1280, |
| "padded_rows": 5120, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 10, |
| "offset": 51453440, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 10, |
| "offset": 58007040, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.26.mlp.mlp.2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 58211840, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.26.mlp.mlp.2.weight": { |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 5120, |
| "padded_rows": 1280, |
| "padded_columns": 5120, |
| "data": { |
| "shard": 10, |
| "offset": 58214400, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 10, |
| "offset": 64768000, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.26.norm2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 64972800, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.26.norm2.weight": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 64975360, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.26.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 64977920, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.26.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 64980480, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.26.self_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 64983040, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.26.self_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 10, |
| "offset": 64985600, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 10, |
| "offset": 66624000, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.26.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3840 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 10, |
| "offset": 66675200, |
| "bytes": 7680 |
| } |
| }, |
| "blocks.26.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3840, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3840, |
| "columns": 1280, |
| "padded_rows": 3840, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 11, |
| "offset": 0, |
| "bytes": 4915200 |
| }, |
| "scales": { |
| "shard": 11, |
| "offset": 4915200, |
| "bytes": 153600 |
| } |
| }, |
| "blocks.27.adaLN_modulation.1.bias": { |
| "shape": [ |
| 7680 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 5068800, |
| "bytes": 15360 |
| } |
| }, |
| "blocks.27.adaLN_modulation.1.weight": { |
| "shape": [ |
| 7680, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 7680, |
| "columns": 1280, |
| "padded_rows": 7680, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 11, |
| "offset": 5084160, |
| "bytes": 9830400 |
| }, |
| "scales": { |
| "shard": 11, |
| "offset": 14914560, |
| "bytes": 307200 |
| } |
| }, |
| "blocks.27.cross_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 15221760, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.27.cross_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 15224320, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.27.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2560 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 15226880, |
| "bytes": 5120 |
| } |
| }, |
| "blocks.27.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2560, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2560, |
| "columns": 768, |
| "padded_rows": 2560, |
| "padded_columns": 768, |
| "data": { |
| "shard": 11, |
| "offset": 15232000, |
| "bytes": 1966080 |
| }, |
| "scales": { |
| "shard": 11, |
| "offset": 17198080, |
| "bytes": 61440 |
| } |
| }, |
| "blocks.27.cross_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 17259520, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.27.cross_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 11, |
| "offset": 17262080, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 11, |
| "offset": 18900480, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.27.cross_attn.to_q.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 18951680, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.27.cross_attn.to_q.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 11, |
| "offset": 18954240, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 11, |
| "offset": 20592640, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.27.mlp.mlp.0.bias": { |
| "shape": [ |
| 5120 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 20643840, |
| "bytes": 10240 |
| } |
| }, |
| "blocks.27.mlp.mlp.0.weight": { |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 5120, |
| "columns": 1280, |
| "padded_rows": 5120, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 11, |
| "offset": 20654080, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 11, |
| "offset": 27207680, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.27.mlp.mlp.2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 27412480, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.27.mlp.mlp.2.weight": { |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 5120, |
| "padded_rows": 1280, |
| "padded_columns": 5120, |
| "data": { |
| "shard": 11, |
| "offset": 27415040, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 11, |
| "offset": 33968640, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.27.norm2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 34173440, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.27.norm2.weight": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 34176000, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.27.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 34178560, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.27.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 34181120, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.27.self_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 34183680, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.27.self_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 11, |
| "offset": 34186240, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 11, |
| "offset": 35824640, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.27.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3840 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 35875840, |
| "bytes": 7680 |
| } |
| }, |
| "blocks.27.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3840, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3840, |
| "columns": 1280, |
| "padded_rows": 3840, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 11, |
| "offset": 35883520, |
| "bytes": 4915200 |
| }, |
| "scales": { |
| "shard": 11, |
| "offset": 40798720, |
| "bytes": 153600 |
| } |
| }, |
| "blocks.3.adaLN_modulation.1.bias": { |
| "shape": [ |
| 7680 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 40952320, |
| "bytes": 15360 |
| } |
| }, |
| "blocks.3.adaLN_modulation.1.weight": { |
| "shape": [ |
| 7680, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 7680, |
| "columns": 1280, |
| "padded_rows": 7680, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 11, |
| "offset": 40967680, |
| "bytes": 9830400 |
| }, |
| "scales": { |
| "shard": 11, |
| "offset": 50798080, |
| "bytes": 307200 |
| } |
| }, |
| "blocks.3.cross_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 51105280, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.3.cross_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 51107840, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.3.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2560 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 51110400, |
| "bytes": 5120 |
| } |
| }, |
| "blocks.3.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2560, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2560, |
| "columns": 768, |
| "padded_rows": 2560, |
| "padded_columns": 768, |
| "data": { |
| "shard": 11, |
| "offset": 51115520, |
| "bytes": 1966080 |
| }, |
| "scales": { |
| "shard": 11, |
| "offset": 53081600, |
| "bytes": 61440 |
| } |
| }, |
| "blocks.3.cross_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 53143040, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.3.cross_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 11, |
| "offset": 53145600, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 11, |
| "offset": 54784000, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.3.cross_attn.to_q.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 54835200, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.3.cross_attn.to_q.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 11, |
| "offset": 54837760, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 11, |
| "offset": 56476160, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.3.mlp.mlp.0.bias": { |
| "shape": [ |
| 5120 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 56527360, |
| "bytes": 10240 |
| } |
| }, |
| "blocks.3.mlp.mlp.0.weight": { |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 5120, |
| "columns": 1280, |
| "padded_rows": 5120, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 11, |
| "offset": 56537600, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 11, |
| "offset": 63091200, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.3.mlp.mlp.2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 11, |
| "offset": 63296000, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.3.mlp.mlp.2.weight": { |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 5120, |
| "padded_rows": 1280, |
| "padded_columns": 5120, |
| "data": { |
| "shard": 12, |
| "offset": 0, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 12, |
| "offset": 6553600, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.3.norm2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 6758400, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.3.norm2.weight": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 6760960, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.3.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 6763520, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.3.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 6766080, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.3.self_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 6768640, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.3.self_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 12, |
| "offset": 6771200, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 12, |
| "offset": 8409600, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.3.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3840 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 8460800, |
| "bytes": 7680 |
| } |
| }, |
| "blocks.3.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3840, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3840, |
| "columns": 1280, |
| "padded_rows": 3840, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 12, |
| "offset": 8468480, |
| "bytes": 4915200 |
| }, |
| "scales": { |
| "shard": 12, |
| "offset": 13383680, |
| "bytes": 153600 |
| } |
| }, |
| "blocks.4.adaLN_modulation.1.bias": { |
| "shape": [ |
| 7680 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 13537280, |
| "bytes": 15360 |
| } |
| }, |
| "blocks.4.adaLN_modulation.1.weight": { |
| "shape": [ |
| 7680, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 7680, |
| "columns": 1280, |
| "padded_rows": 7680, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 12, |
| "offset": 13552640, |
| "bytes": 9830400 |
| }, |
| "scales": { |
| "shard": 12, |
| "offset": 23383040, |
| "bytes": 307200 |
| } |
| }, |
| "blocks.4.cross_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 23690240, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.4.cross_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 23692800, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.4.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2560 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 23695360, |
| "bytes": 5120 |
| } |
| }, |
| "blocks.4.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2560, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2560, |
| "columns": 768, |
| "padded_rows": 2560, |
| "padded_columns": 768, |
| "data": { |
| "shard": 12, |
| "offset": 23700480, |
| "bytes": 1966080 |
| }, |
| "scales": { |
| "shard": 12, |
| "offset": 25666560, |
| "bytes": 61440 |
| } |
| }, |
| "blocks.4.cross_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 25728000, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.4.cross_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 12, |
| "offset": 25730560, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 12, |
| "offset": 27368960, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.4.cross_attn.to_q.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 27420160, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.4.cross_attn.to_q.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 12, |
| "offset": 27422720, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 12, |
| "offset": 29061120, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.4.mlp.mlp.0.bias": { |
| "shape": [ |
| 5120 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 29112320, |
| "bytes": 10240 |
| } |
| }, |
| "blocks.4.mlp.mlp.0.weight": { |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 5120, |
| "columns": 1280, |
| "padded_rows": 5120, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 12, |
| "offset": 29122560, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 12, |
| "offset": 35676160, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.4.mlp.mlp.2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 35880960, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.4.mlp.mlp.2.weight": { |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 5120, |
| "padded_rows": 1280, |
| "padded_columns": 5120, |
| "data": { |
| "shard": 12, |
| "offset": 35883520, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 12, |
| "offset": 42437120, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.4.norm2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 42641920, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.4.norm2.weight": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 42644480, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.4.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 42647040, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.4.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 42649600, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.4.self_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 42652160, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.4.self_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 12, |
| "offset": 42654720, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 12, |
| "offset": 44293120, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.4.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3840 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 44344320, |
| "bytes": 7680 |
| } |
| }, |
| "blocks.4.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3840, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3840, |
| "columns": 1280, |
| "padded_rows": 3840, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 12, |
| "offset": 44352000, |
| "bytes": 4915200 |
| }, |
| "scales": { |
| "shard": 12, |
| "offset": 49267200, |
| "bytes": 153600 |
| } |
| }, |
| "blocks.5.adaLN_modulation.1.bias": { |
| "shape": [ |
| 7680 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 49420800, |
| "bytes": 15360 |
| } |
| }, |
| "blocks.5.adaLN_modulation.1.weight": { |
| "shape": [ |
| 7680, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 7680, |
| "columns": 1280, |
| "padded_rows": 7680, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 12, |
| "offset": 49436160, |
| "bytes": 9830400 |
| }, |
| "scales": { |
| "shard": 12, |
| "offset": 59266560, |
| "bytes": 307200 |
| } |
| }, |
| "blocks.5.cross_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 59573760, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.5.cross_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 59576320, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.5.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2560 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 59578880, |
| "bytes": 5120 |
| } |
| }, |
| "blocks.5.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2560, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2560, |
| "columns": 768, |
| "padded_rows": 2560, |
| "padded_columns": 768, |
| "data": { |
| "shard": 12, |
| "offset": 59584000, |
| "bytes": 1966080 |
| }, |
| "scales": { |
| "shard": 12, |
| "offset": 61550080, |
| "bytes": 61440 |
| } |
| }, |
| "blocks.5.cross_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 61611520, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.5.cross_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 12, |
| "offset": 61614080, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 12, |
| "offset": 63252480, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.5.cross_attn.to_q.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 63303680, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.5.cross_attn.to_q.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 12, |
| "offset": 63306240, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 12, |
| "offset": 64944640, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.5.mlp.mlp.0.bias": { |
| "shape": [ |
| 5120 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 12, |
| "offset": 64995840, |
| "bytes": 10240 |
| } |
| }, |
| "blocks.5.mlp.mlp.0.weight": { |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 5120, |
| "columns": 1280, |
| "padded_rows": 5120, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 13, |
| "offset": 0, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 13, |
| "offset": 6553600, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.5.mlp.mlp.2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 6758400, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.5.mlp.mlp.2.weight": { |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 5120, |
| "padded_rows": 1280, |
| "padded_columns": 5120, |
| "data": { |
| "shard": 13, |
| "offset": 6760960, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 13, |
| "offset": 13314560, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.5.norm2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 13519360, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.5.norm2.weight": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 13521920, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.5.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 13524480, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.5.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 13527040, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.5.self_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 13529600, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.5.self_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 13, |
| "offset": 13532160, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 13, |
| "offset": 15170560, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.5.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3840 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 15221760, |
| "bytes": 7680 |
| } |
| }, |
| "blocks.5.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3840, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3840, |
| "columns": 1280, |
| "padded_rows": 3840, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 13, |
| "offset": 15229440, |
| "bytes": 4915200 |
| }, |
| "scales": { |
| "shard": 13, |
| "offset": 20144640, |
| "bytes": 153600 |
| } |
| }, |
| "blocks.6.adaLN_modulation.1.bias": { |
| "shape": [ |
| 7680 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 20298240, |
| "bytes": 15360 |
| } |
| }, |
| "blocks.6.adaLN_modulation.1.weight": { |
| "shape": [ |
| 7680, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 7680, |
| "columns": 1280, |
| "padded_rows": 7680, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 13, |
| "offset": 20313600, |
| "bytes": 9830400 |
| }, |
| "scales": { |
| "shard": 13, |
| "offset": 30144000, |
| "bytes": 307200 |
| } |
| }, |
| "blocks.6.cross_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 30451200, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.6.cross_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 30453760, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.6.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2560 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 30456320, |
| "bytes": 5120 |
| } |
| }, |
| "blocks.6.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2560, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2560, |
| "columns": 768, |
| "padded_rows": 2560, |
| "padded_columns": 768, |
| "data": { |
| "shard": 13, |
| "offset": 30461440, |
| "bytes": 1966080 |
| }, |
| "scales": { |
| "shard": 13, |
| "offset": 32427520, |
| "bytes": 61440 |
| } |
| }, |
| "blocks.6.cross_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 32488960, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.6.cross_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 13, |
| "offset": 32491520, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 13, |
| "offset": 34129920, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.6.cross_attn.to_q.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 34181120, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.6.cross_attn.to_q.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 13, |
| "offset": 34183680, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 13, |
| "offset": 35822080, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.6.mlp.mlp.0.bias": { |
| "shape": [ |
| 5120 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 35873280, |
| "bytes": 10240 |
| } |
| }, |
| "blocks.6.mlp.mlp.0.weight": { |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 5120, |
| "columns": 1280, |
| "padded_rows": 5120, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 13, |
| "offset": 35883520, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 13, |
| "offset": 42437120, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.6.mlp.mlp.2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 42641920, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.6.mlp.mlp.2.weight": { |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 5120, |
| "padded_rows": 1280, |
| "padded_columns": 5120, |
| "data": { |
| "shard": 13, |
| "offset": 42644480, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 13, |
| "offset": 49198080, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.6.norm2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 49402880, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.6.norm2.weight": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 49405440, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.6.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 49408000, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.6.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 49410560, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.6.self_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 49413120, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.6.self_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 13, |
| "offset": 49415680, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 13, |
| "offset": 51054080, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.6.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3840 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 51105280, |
| "bytes": 7680 |
| } |
| }, |
| "blocks.6.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3840, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3840, |
| "columns": 1280, |
| "padded_rows": 3840, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 13, |
| "offset": 51112960, |
| "bytes": 4915200 |
| }, |
| "scales": { |
| "shard": 13, |
| "offset": 56028160, |
| "bytes": 153600 |
| } |
| }, |
| "blocks.7.adaLN_modulation.1.bias": { |
| "shape": [ |
| 7680 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 56181760, |
| "bytes": 15360 |
| } |
| }, |
| "blocks.7.adaLN_modulation.1.weight": { |
| "shape": [ |
| 7680, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 7680, |
| "columns": 1280, |
| "padded_rows": 7680, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 13, |
| "offset": 56197120, |
| "bytes": 9830400 |
| }, |
| "scales": { |
| "shard": 13, |
| "offset": 66027520, |
| "bytes": 307200 |
| } |
| }, |
| "blocks.7.cross_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 66334720, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.7.cross_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 66337280, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.7.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2560 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 13, |
| "offset": 66339840, |
| "bytes": 5120 |
| } |
| }, |
| "blocks.7.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2560, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2560, |
| "columns": 768, |
| "padded_rows": 2560, |
| "padded_columns": 768, |
| "data": { |
| "shard": 14, |
| "offset": 0, |
| "bytes": 1966080 |
| }, |
| "scales": { |
| "shard": 14, |
| "offset": 1966080, |
| "bytes": 61440 |
| } |
| }, |
| "blocks.7.cross_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 14, |
| "offset": 2027520, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.7.cross_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 14, |
| "offset": 2030080, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 14, |
| "offset": 3668480, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.7.cross_attn.to_q.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 14, |
| "offset": 3719680, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.7.cross_attn.to_q.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 14, |
| "offset": 3722240, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 14, |
| "offset": 5360640, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.7.mlp.mlp.0.bias": { |
| "shape": [ |
| 5120 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 14, |
| "offset": 5411840, |
| "bytes": 10240 |
| } |
| }, |
| "blocks.7.mlp.mlp.0.weight": { |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 5120, |
| "columns": 1280, |
| "padded_rows": 5120, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 14, |
| "offset": 5422080, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 14, |
| "offset": 11975680, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.7.mlp.mlp.2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 14, |
| "offset": 12180480, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.7.mlp.mlp.2.weight": { |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 5120, |
| "padded_rows": 1280, |
| "padded_columns": 5120, |
| "data": { |
| "shard": 14, |
| "offset": 12183040, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 14, |
| "offset": 18736640, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.7.norm2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 14, |
| "offset": 18941440, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.7.norm2.weight": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 14, |
| "offset": 18944000, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.7.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 14, |
| "offset": 18946560, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.7.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 14, |
| "offset": 18949120, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.7.self_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 14, |
| "offset": 18951680, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.7.self_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 14, |
| "offset": 18954240, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 14, |
| "offset": 20592640, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.7.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3840 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 14, |
| "offset": 20643840, |
| "bytes": 7680 |
| } |
| }, |
| "blocks.7.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3840, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3840, |
| "columns": 1280, |
| "padded_rows": 3840, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 14, |
| "offset": 20651520, |
| "bytes": 4915200 |
| }, |
| "scales": { |
| "shard": 14, |
| "offset": 25566720, |
| "bytes": 153600 |
| } |
| }, |
| "blocks.8.adaLN_modulation.1.bias": { |
| "shape": [ |
| 7680 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 14, |
| "offset": 25720320, |
| "bytes": 15360 |
| } |
| }, |
| "blocks.8.adaLN_modulation.1.weight": { |
| "shape": [ |
| 7680, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 7680, |
| "columns": 1280, |
| "padded_rows": 7680, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 14, |
| "offset": 25735680, |
| "bytes": 9830400 |
| }, |
| "scales": { |
| "shard": 14, |
| "offset": 35566080, |
| "bytes": 307200 |
| } |
| }, |
| "blocks.8.cross_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 14, |
| "offset": 35873280, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.8.cross_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 14, |
| "offset": 35875840, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.8.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2560 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 14, |
| "offset": 35878400, |
| "bytes": 5120 |
| } |
| }, |
| "blocks.8.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2560, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2560, |
| "columns": 768, |
| "padded_rows": 2560, |
| "padded_columns": 768, |
| "data": { |
| "shard": 14, |
| "offset": 35883520, |
| "bytes": 1966080 |
| }, |
| "scales": { |
| "shard": 14, |
| "offset": 37849600, |
| "bytes": 61440 |
| } |
| }, |
| "blocks.8.cross_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 14, |
| "offset": 37911040, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.8.cross_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 14, |
| "offset": 37913600, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 14, |
| "offset": 39552000, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.8.cross_attn.to_q.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 14, |
| "offset": 39603200, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.8.cross_attn.to_q.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 14, |
| "offset": 39605760, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 14, |
| "offset": 41244160, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.8.mlp.mlp.0.bias": { |
| "shape": [ |
| 5120 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 14, |
| "offset": 41295360, |
| "bytes": 10240 |
| } |
| }, |
| "blocks.8.mlp.mlp.0.weight": { |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 5120, |
| "columns": 1280, |
| "padded_rows": 5120, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 14, |
| "offset": 41305600, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 14, |
| "offset": 47859200, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.8.mlp.mlp.2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 14, |
| "offset": 48064000, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.8.mlp.mlp.2.weight": { |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 5120, |
| "padded_rows": 1280, |
| "padded_columns": 5120, |
| "data": { |
| "shard": 14, |
| "offset": 48066560, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 14, |
| "offset": 54620160, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.8.norm2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 14, |
| "offset": 54824960, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.8.norm2.weight": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 14, |
| "offset": 54827520, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.8.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 14, |
| "offset": 54830080, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.8.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 14, |
| "offset": 54832640, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.8.self_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 14, |
| "offset": 54835200, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.8.self_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 14, |
| "offset": 54837760, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 14, |
| "offset": 56476160, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.8.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3840 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 14, |
| "offset": 56527360, |
| "bytes": 7680 |
| } |
| }, |
| "blocks.8.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3840, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3840, |
| "columns": 1280, |
| "padded_rows": 3840, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 14, |
| "offset": 56535040, |
| "bytes": 4915200 |
| }, |
| "scales": { |
| "shard": 14, |
| "offset": 61450240, |
| "bytes": 153600 |
| } |
| }, |
| "blocks.9.adaLN_modulation.1.bias": { |
| "shape": [ |
| 7680 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 14, |
| "offset": 61603840, |
| "bytes": 15360 |
| } |
| }, |
| "blocks.9.adaLN_modulation.1.weight": { |
| "shape": [ |
| 7680, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 7680, |
| "columns": 1280, |
| "padded_rows": 7680, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 15, |
| "offset": 0, |
| "bytes": 9830400 |
| }, |
| "scales": { |
| "shard": 15, |
| "offset": 9830400, |
| "bytes": 307200 |
| } |
| }, |
| "blocks.9.cross_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 15, |
| "offset": 10137600, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.9.cross_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 15, |
| "offset": 10140160, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.9.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2560 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 15, |
| "offset": 10142720, |
| "bytes": 5120 |
| } |
| }, |
| "blocks.9.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2560, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2560, |
| "columns": 768, |
| "padded_rows": 2560, |
| "padded_columns": 768, |
| "data": { |
| "shard": 15, |
| "offset": 10147840, |
| "bytes": 1966080 |
| }, |
| "scales": { |
| "shard": 15, |
| "offset": 12113920, |
| "bytes": 61440 |
| } |
| }, |
| "blocks.9.cross_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 15, |
| "offset": 12175360, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.9.cross_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 15, |
| "offset": 12177920, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 15, |
| "offset": 13816320, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.9.cross_attn.to_q.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 15, |
| "offset": 13867520, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.9.cross_attn.to_q.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 15, |
| "offset": 13870080, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 15, |
| "offset": 15508480, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.9.mlp.mlp.0.bias": { |
| "shape": [ |
| 5120 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 15, |
| "offset": 15559680, |
| "bytes": 10240 |
| } |
| }, |
| "blocks.9.mlp.mlp.0.weight": { |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 5120, |
| "columns": 1280, |
| "padded_rows": 5120, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 15, |
| "offset": 15569920, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 15, |
| "offset": 22123520, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.9.mlp.mlp.2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 15, |
| "offset": 22328320, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.9.mlp.mlp.2.weight": { |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 5120, |
| "padded_rows": 1280, |
| "padded_columns": 5120, |
| "data": { |
| "shard": 15, |
| "offset": 22330880, |
| "bytes": 6553600 |
| }, |
| "scales": { |
| "shard": 15, |
| "offset": 28884480, |
| "bytes": 204800 |
| } |
| }, |
| "blocks.9.norm2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 15, |
| "offset": 29089280, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.9.norm2.weight": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 15, |
| "offset": 29091840, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.9.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 15, |
| "offset": 29094400, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.9.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 80 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 15, |
| "offset": 29096960, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.9.self_attn.to_out.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 15, |
| "offset": 29099520, |
| "bytes": 2560 |
| } |
| }, |
| "blocks.9.self_attn.to_out.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 15, |
| "offset": 29102080, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 15, |
| "offset": 30740480, |
| "bytes": 51200 |
| } |
| }, |
| "blocks.9.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3840 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 15, |
| "offset": 30791680, |
| "bytes": 7680 |
| } |
| }, |
| "blocks.9.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3840, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3840, |
| "columns": 1280, |
| "padded_rows": 3840, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 15, |
| "offset": 30799360, |
| "bytes": 4915200 |
| }, |
| "scales": { |
| "shard": 15, |
| "offset": 35714560, |
| "bytes": 153600 |
| } |
| }, |
| "input_layer.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 15, |
| "offset": 35868160, |
| "bytes": 2560 |
| } |
| }, |
| "input_layer.weight": { |
| "shape": [ |
| 1280, |
| 8 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 8, |
| "padded_rows": 1280, |
| "padded_columns": 64, |
| "data": { |
| "shard": 15, |
| "offset": 35870720, |
| "bytes": 81920 |
| }, |
| "scales": { |
| "shard": 15, |
| "offset": 35952640, |
| "bytes": 2560 |
| } |
| }, |
| "out_layer.bias": { |
| "shape": [ |
| 8 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 15, |
| "offset": 35955200, |
| "bytes": 16 |
| } |
| }, |
| "out_layer.weight": { |
| "shape": [ |
| 8, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 8, |
| "columns": 1280, |
| "padded_rows": 16, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 15, |
| "offset": 35955456, |
| "bytes": 20480 |
| }, |
| "scales": { |
| "shard": 15, |
| "offset": 35975936, |
| "bytes": 640 |
| } |
| }, |
| "pos_emb": { |
| "shape": [ |
| 4096, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1280, |
| "padded_rows": 4096, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 15, |
| "offset": 35976704, |
| "bytes": 5242880 |
| }, |
| "scales": { |
| "shard": 15, |
| "offset": 41219584, |
| "bytes": 163840 |
| } |
| }, |
| "t_embedder.mlp.0.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 15, |
| "offset": 41383424, |
| "bytes": 2560 |
| } |
| }, |
| "t_embedder.mlp.0.weight": { |
| "shape": [ |
| 1280, |
| 256 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 256, |
| "padded_rows": 1280, |
| "padded_columns": 256, |
| "data": { |
| "shard": 15, |
| "offset": 41385984, |
| "bytes": 327680 |
| }, |
| "scales": { |
| "shard": 15, |
| "offset": 41713664, |
| "bytes": 10240 |
| } |
| }, |
| "t_embedder.mlp.2.bias": { |
| "shape": [ |
| 1280 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 15, |
| "offset": 41723904, |
| "bytes": 2560 |
| } |
| }, |
| "t_embedder.mlp.2.weight": { |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1280, |
| "columns": 1280, |
| "padded_rows": 1280, |
| "padded_columns": 1280, |
| "data": { |
| "shard": 15, |
| "offset": 41726464, |
| "bytes": 1638400 |
| }, |
| "scales": { |
| "shard": 15, |
| "offset": 43364864, |
| "bytes": 51200 |
| } |
| } |
| } |
| } |
|
|
