{ "schema": "trellis-webgpu-pack", "version": 1, "stage": "sparseStructureFlow", "upstreamRevision": "4aad9f4a110329a410974d7f41ce5333a9a1fc87", "shards": [ { "file": "stage-000.bin", "bytes": 66643968, "sha256": "05087b26ad671d96adae17ad09d6ad30b97925440027c51f91e2bf511af93e77" }, { "file": "stage-001.bin", "bytes": 64464896, "sha256": "2e2e70521febf08cd1869e606ed38a5d2f20d539d05e4188490f79c59a674e47" }, { "file": "stage-002.bin", "bytes": 65560576, "sha256": "a54270c6ad310a93267260f07651ba7064b8477f1d58028a4f848089241c82bb" }, { "file": "stage-003.bin", "bytes": 66091008, "sha256": "fb2c2fba4108b9eb0e099c207b19d0de99aed1eca3436d082282aab1483cb323" }, { "file": "stage-004.bin", "bytes": 63395840, "sha256": "ba5d1e2802670282240691d2b5cab092ddc3a0d62f8beac1890f18f2bc812aee" }, { "file": "stage-005.bin", "bytes": 66631680, "sha256": "4a64c79811c881cbde281b13ae11a3e8375902cead1126f07dd62e422da5de07" }, { "file": "stage-006.bin", "bytes": 64464896, "sha256": "a5dc01ce0eae158edcb07fb7d63865f97c35c87a58ac0f565356126551672984" }, { "file": "stage-007.bin", "bytes": 65560576, "sha256": "fbd8d6a918fca4718f50fe296ca69710b22826180790030bd735e040a4762a2b" }, { "file": "stage-008.bin", "bytes": 42058496, "sha256": "126a3f3b2e704bad719342ad826ac774fc9b176b84eda7ec8a7e2234958b57c9" } ], "tensors": { "blocks.0.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 0, "bytes": 12288 } }, "blocks.0.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 0, "offset": 12288, "bytes": 6291456 }, "scales": { "shard": 0, "offset": 6303744, "bytes": 196608 } }, "blocks.0.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 6500352, "bytes": 4096 } }, "blocks.0.cross_attn.to_kv.weight": { "shape": [ 2048, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 768, "padded_rows": 2048, "padded_columns": 768, "data": { "shard": 0, "offset": 6504448, "bytes": 1572864 }, "scales": { "shard": 0, "offset": 8077312, "bytes": 49152 } }, "blocks.0.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 8126464, "bytes": 2048 } }, "blocks.0.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 0, "offset": 8128512, "bytes": 1048576 }, "scales": { "shard": 0, "offset": 9177088, "bytes": 32768 } }, "blocks.0.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 9209856, "bytes": 2048 } }, "blocks.0.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 0, "offset": 9211904, "bytes": 1048576 }, "scales": { "shard": 0, "offset": 10260480, "bytes": 32768 } }, "blocks.0.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 10293248, "bytes": 8192 } }, "blocks.0.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 0, "offset": 10301440, "bytes": 4194304 }, "scales": { "shard": 0, "offset": 14495744, "bytes": 131072 } }, "blocks.0.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 14626816, "bytes": 2048 } }, "blocks.0.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 0, "offset": 14628864, "bytes": 4194304 }, "scales": { "shard": 0, "offset": 18823168, "bytes": 131072 } }, "blocks.0.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 18954240, "bytes": 2048 } }, "blocks.0.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 18956288, "bytes": 2048 } }, "blocks.0.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 18958336, "bytes": 2048 } }, "blocks.0.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 18960384, "bytes": 2048 } }, "blocks.0.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 18962432, "bytes": 2048 } }, "blocks.0.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 0, "offset": 18964480, "bytes": 1048576 }, "scales": { "shard": 0, "offset": 20013056, "bytes": 32768 } }, "blocks.0.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 20045824, "bytes": 6144 } }, "blocks.0.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 0, "offset": 20051968, "bytes": 3145728 }, "scales": { "shard": 0, "offset": 23197696, "bytes": 98304 } }, "blocks.1.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 23296000, "bytes": 12288 } }, "blocks.1.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 0, "offset": 23308288, "bytes": 6291456 }, "scales": { "shard": 0, "offset": 29599744, "bytes": 196608 } }, "blocks.1.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 29796352, "bytes": 4096 } }, "blocks.1.cross_attn.to_kv.weight": { "shape": [ 2048, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 768, "padded_rows": 2048, "padded_columns": 768, "data": { "shard": 0, "offset": 29800448, "bytes": 1572864 }, "scales": { "shard": 0, "offset": 31373312, "bytes": 49152 } }, "blocks.1.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 31422464, "bytes": 2048 } }, "blocks.1.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 0, "offset": 31424512, "bytes": 1048576 }, "scales": { "shard": 0, "offset": 32473088, "bytes": 32768 } }, "blocks.1.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 32505856, "bytes": 2048 } }, "blocks.1.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 0, "offset": 32507904, "bytes": 1048576 }, "scales": { "shard": 0, "offset": 33556480, "bytes": 32768 } }, "blocks.1.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 33589248, "bytes": 8192 } }, "blocks.1.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 0, "offset": 33597440, "bytes": 4194304 }, "scales": { "shard": 0, "offset": 37791744, "bytes": 131072 } }, "blocks.1.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 37922816, "bytes": 2048 } }, "blocks.1.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 0, "offset": 37924864, "bytes": 4194304 }, "scales": { "shard": 0, "offset": 42119168, "bytes": 131072 } }, "blocks.1.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 42250240, "bytes": 2048 } }, "blocks.1.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 42252288, "bytes": 2048 } }, "blocks.1.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 42254336, "bytes": 2048 } }, "blocks.1.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 42256384, "bytes": 2048 } }, "blocks.1.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 42258432, "bytes": 2048 } }, "blocks.1.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 0, "offset": 42260480, "bytes": 1048576 }, "scales": { "shard": 0, "offset": 43309056, "bytes": 32768 } }, "blocks.1.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 43341824, "bytes": 6144 } }, "blocks.1.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 0, "offset": 43347968, "bytes": 3145728 }, "scales": { "shard": 0, "offset": 46493696, "bytes": 98304 } }, "blocks.10.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 46592000, "bytes": 12288 } }, "blocks.10.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 0, "offset": 46604288, "bytes": 6291456 }, "scales": { "shard": 0, "offset": 52895744, "bytes": 196608 } }, "blocks.10.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 53092352, "bytes": 4096 } }, "blocks.10.cross_attn.to_kv.weight": { "shape": [ 2048, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 768, "padded_rows": 2048, "padded_columns": 768, "data": { "shard": 0, "offset": 53096448, "bytes": 1572864 }, "scales": { "shard": 0, "offset": 54669312, "bytes": 49152 } }, "blocks.10.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 54718464, "bytes": 2048 } }, "blocks.10.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 0, "offset": 54720512, "bytes": 1048576 }, "scales": { "shard": 0, "offset": 55769088, "bytes": 32768 } }, "blocks.10.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 55801856, "bytes": 2048 } }, "blocks.10.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 0, "offset": 55803904, "bytes": 1048576 }, "scales": { "shard": 0, "offset": 56852480, "bytes": 32768 } }, "blocks.10.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 56885248, "bytes": 8192 } }, "blocks.10.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 0, "offset": 56893440, "bytes": 4194304 }, "scales": { "shard": 0, "offset": 61087744, "bytes": 131072 } }, "blocks.10.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 61218816, "bytes": 2048 } }, "blocks.10.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 0, "offset": 61220864, "bytes": 4194304 }, "scales": { "shard": 0, "offset": 65415168, "bytes": 131072 } }, "blocks.10.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 65546240, "bytes": 2048 } }, "blocks.10.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 65548288, "bytes": 2048 } }, "blocks.10.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 65550336, "bytes": 2048 } }, "blocks.10.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 65552384, "bytes": 2048 } }, "blocks.10.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 65554432, "bytes": 2048 } }, "blocks.10.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 0, "offset": 65556480, "bytes": 1048576 }, "scales": { "shard": 0, "offset": 66605056, "bytes": 32768 } }, "blocks.10.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 0, "offset": 66637824, "bytes": 6144 } }, "blocks.10.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 1, "offset": 0, "bytes": 3145728 }, "scales": { "shard": 1, "offset": 3145728, "bytes": 98304 } }, "blocks.11.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 3244032, "bytes": 12288 } }, "blocks.11.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 1, "offset": 3256320, "bytes": 6291456 }, "scales": { "shard": 1, "offset": 9547776, "bytes": 196608 } }, "blocks.11.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 9744384, "bytes": 4096 } }, "blocks.11.cross_attn.to_kv.weight": { "shape": [ 2048, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 768, "padded_rows": 2048, "padded_columns": 768, "data": { "shard": 1, "offset": 9748480, "bytes": 1572864 }, "scales": { "shard": 1, "offset": 11321344, "bytes": 49152 } }, "blocks.11.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 11370496, "bytes": 2048 } }, "blocks.11.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 1, "offset": 11372544, "bytes": 1048576 }, "scales": { "shard": 1, "offset": 12421120, "bytes": 32768 } }, "blocks.11.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 12453888, "bytes": 2048 } }, "blocks.11.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 1, "offset": 12455936, "bytes": 1048576 }, "scales": { "shard": 1, "offset": 13504512, "bytes": 32768 } }, "blocks.11.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 13537280, "bytes": 8192 } }, "blocks.11.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 1, "offset": 13545472, "bytes": 4194304 }, "scales": { "shard": 1, "offset": 17739776, "bytes": 131072 } }, "blocks.11.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 17870848, "bytes": 2048 } }, "blocks.11.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 1, "offset": 17872896, "bytes": 4194304 }, "scales": { "shard": 1, "offset": 22067200, "bytes": 131072 } }, "blocks.11.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 22198272, "bytes": 2048 } }, "blocks.11.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 22200320, "bytes": 2048 } }, "blocks.11.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 22202368, "bytes": 2048 } }, "blocks.11.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 22204416, "bytes": 2048 } }, "blocks.11.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 22206464, "bytes": 2048 } }, "blocks.11.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 1, "offset": 22208512, "bytes": 1048576 }, "scales": { "shard": 1, "offset": 23257088, "bytes": 32768 } }, "blocks.11.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 23289856, "bytes": 6144 } }, "blocks.11.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 1, "offset": 23296000, "bytes": 3145728 }, "scales": { "shard": 1, "offset": 26441728, "bytes": 98304 } }, "blocks.12.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 26540032, "bytes": 12288 } }, "blocks.12.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 1, "offset": 26552320, "bytes": 6291456 }, "scales": { "shard": 1, "offset": 32843776, "bytes": 196608 } }, "blocks.12.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 33040384, "bytes": 4096 } }, "blocks.12.cross_attn.to_kv.weight": { "shape": [ 2048, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 768, "padded_rows": 2048, "padded_columns": 768, "data": { "shard": 1, "offset": 33044480, "bytes": 1572864 }, "scales": { "shard": 1, "offset": 34617344, "bytes": 49152 } }, "blocks.12.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 34666496, "bytes": 2048 } }, "blocks.12.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 1, "offset": 34668544, "bytes": 1048576 }, "scales": { "shard": 1, "offset": 35717120, "bytes": 32768 } }, "blocks.12.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 35749888, "bytes": 2048 } }, "blocks.12.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 1, "offset": 35751936, "bytes": 1048576 }, "scales": { "shard": 1, "offset": 36800512, "bytes": 32768 } }, "blocks.12.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 36833280, "bytes": 8192 } }, "blocks.12.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 1, "offset": 36841472, "bytes": 4194304 }, "scales": { "shard": 1, "offset": 41035776, "bytes": 131072 } }, "blocks.12.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 41166848, "bytes": 2048 } }, "blocks.12.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 1, "offset": 41168896, "bytes": 4194304 }, "scales": { "shard": 1, "offset": 45363200, "bytes": 131072 } }, "blocks.12.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 45494272, "bytes": 2048 } }, "blocks.12.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 45496320, "bytes": 2048 } }, "blocks.12.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 45498368, "bytes": 2048 } }, "blocks.12.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 45500416, "bytes": 2048 } }, "blocks.12.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 45502464, "bytes": 2048 } }, "blocks.12.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 1, "offset": 45504512, "bytes": 1048576 }, "scales": { "shard": 1, "offset": 46553088, "bytes": 32768 } }, "blocks.12.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 46585856, "bytes": 6144 } }, "blocks.12.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 1, "offset": 46592000, "bytes": 3145728 }, "scales": { "shard": 1, "offset": 49737728, "bytes": 98304 } }, "blocks.13.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 49836032, "bytes": 12288 } }, "blocks.13.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 1, "offset": 49848320, "bytes": 6291456 }, "scales": { "shard": 1, "offset": 56139776, "bytes": 196608 } }, "blocks.13.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 56336384, "bytes": 4096 } }, "blocks.13.cross_attn.to_kv.weight": { "shape": [ 2048, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 768, "padded_rows": 2048, "padded_columns": 768, "data": { "shard": 1, "offset": 56340480, "bytes": 1572864 }, "scales": { "shard": 1, "offset": 57913344, "bytes": 49152 } }, "blocks.13.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 57962496, "bytes": 2048 } }, "blocks.13.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 1, "offset": 57964544, "bytes": 1048576 }, "scales": { "shard": 1, "offset": 59013120, "bytes": 32768 } }, "blocks.13.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 59045888, "bytes": 2048 } }, "blocks.13.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 1, "offset": 59047936, "bytes": 1048576 }, "scales": { "shard": 1, "offset": 60096512, "bytes": 32768 } }, "blocks.13.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 60129280, "bytes": 8192 } }, "blocks.13.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 1, "offset": 60137472, "bytes": 4194304 }, "scales": { "shard": 1, "offset": 64331776, "bytes": 131072 } }, "blocks.13.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 1, "offset": 64462848, "bytes": 2048 } }, "blocks.13.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 2, "offset": 0, "bytes": 4194304 }, "scales": { "shard": 2, "offset": 4194304, "bytes": 131072 } }, "blocks.13.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 4325376, "bytes": 2048 } }, "blocks.13.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 4327424, "bytes": 2048 } }, "blocks.13.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 4329472, "bytes": 2048 } }, "blocks.13.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 4331520, "bytes": 2048 } }, "blocks.13.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 4333568, "bytes": 2048 } }, "blocks.13.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 2, "offset": 4335616, "bytes": 1048576 }, "scales": { "shard": 2, "offset": 5384192, "bytes": 32768 } }, "blocks.13.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 5416960, "bytes": 6144 } }, "blocks.13.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 2, "offset": 5423104, "bytes": 3145728 }, "scales": { "shard": 2, "offset": 8568832, "bytes": 98304 } }, "blocks.14.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 8667136, "bytes": 12288 } }, "blocks.14.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 2, "offset": 8679424, "bytes": 6291456 }, "scales": { "shard": 2, "offset": 14970880, "bytes": 196608 } }, "blocks.14.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 15167488, "bytes": 4096 } }, "blocks.14.cross_attn.to_kv.weight": { "shape": [ 2048, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 768, "padded_rows": 2048, "padded_columns": 768, "data": { "shard": 2, "offset": 15171584, "bytes": 1572864 }, "scales": { "shard": 2, "offset": 16744448, "bytes": 49152 } }, "blocks.14.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 16793600, "bytes": 2048 } }, "blocks.14.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 2, "offset": 16795648, "bytes": 1048576 }, "scales": { "shard": 2, "offset": 17844224, "bytes": 32768 } }, "blocks.14.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 17876992, "bytes": 2048 } }, "blocks.14.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 2, "offset": 17879040, "bytes": 1048576 }, "scales": { "shard": 2, "offset": 18927616, "bytes": 32768 } }, "blocks.14.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 18960384, "bytes": 8192 } }, "blocks.14.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 2, "offset": 18968576, "bytes": 4194304 }, "scales": { "shard": 2, "offset": 23162880, "bytes": 131072 } }, "blocks.14.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 23293952, "bytes": 2048 } }, "blocks.14.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 2, "offset": 23296000, "bytes": 4194304 }, "scales": { "shard": 2, "offset": 27490304, "bytes": 131072 } }, "blocks.14.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 27621376, "bytes": 2048 } }, "blocks.14.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 27623424, "bytes": 2048 } }, "blocks.14.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 27625472, "bytes": 2048 } }, "blocks.14.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 27627520, "bytes": 2048 } }, "blocks.14.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 27629568, "bytes": 2048 } }, "blocks.14.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 2, "offset": 27631616, "bytes": 1048576 }, "scales": { "shard": 2, "offset": 28680192, "bytes": 32768 } }, "blocks.14.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 28712960, "bytes": 6144 } }, "blocks.14.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 2, "offset": 28719104, "bytes": 3145728 }, "scales": { "shard": 2, "offset": 31864832, "bytes": 98304 } }, "blocks.15.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 31963136, "bytes": 12288 } }, "blocks.15.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 2, "offset": 31975424, "bytes": 6291456 }, "scales": { "shard": 2, "offset": 38266880, "bytes": 196608 } }, "blocks.15.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 38463488, "bytes": 4096 } }, "blocks.15.cross_attn.to_kv.weight": { "shape": [ 2048, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 768, "padded_rows": 2048, "padded_columns": 768, "data": { "shard": 2, "offset": 38467584, "bytes": 1572864 }, "scales": { "shard": 2, "offset": 40040448, "bytes": 49152 } }, "blocks.15.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 40089600, "bytes": 2048 } }, "blocks.15.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 2, "offset": 40091648, "bytes": 1048576 }, "scales": { "shard": 2, "offset": 41140224, "bytes": 32768 } }, "blocks.15.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 41172992, "bytes": 2048 } }, "blocks.15.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 2, "offset": 41175040, "bytes": 1048576 }, "scales": { "shard": 2, "offset": 42223616, "bytes": 32768 } }, "blocks.15.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 42256384, "bytes": 8192 } }, "blocks.15.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 2, "offset": 42264576, "bytes": 4194304 }, "scales": { "shard": 2, "offset": 46458880, "bytes": 131072 } }, "blocks.15.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 46589952, "bytes": 2048 } }, "blocks.15.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 2, "offset": 46592000, "bytes": 4194304 }, "scales": { "shard": 2, "offset": 50786304, "bytes": 131072 } }, "blocks.15.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 50917376, "bytes": 2048 } }, "blocks.15.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 50919424, "bytes": 2048 } }, "blocks.15.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 50921472, "bytes": 2048 } }, "blocks.15.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 50923520, "bytes": 2048 } }, "blocks.15.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 50925568, "bytes": 2048 } }, "blocks.15.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 2, "offset": 50927616, "bytes": 1048576 }, "scales": { "shard": 2, "offset": 51976192, "bytes": 32768 } }, "blocks.15.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 52008960, "bytes": 6144 } }, "blocks.15.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 2, "offset": 52015104, "bytes": 3145728 }, "scales": { "shard": 2, "offset": 55160832, "bytes": 98304 } }, "blocks.16.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 55259136, "bytes": 12288 } }, "blocks.16.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 2, "offset": 55271424, "bytes": 6291456 }, "scales": { "shard": 2, "offset": 61562880, "bytes": 196608 } }, "blocks.16.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 61759488, "bytes": 4096 } }, "blocks.16.cross_attn.to_kv.weight": { "shape": [ 2048, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 768, "padded_rows": 2048, "padded_columns": 768, "data": { "shard": 2, "offset": 61763584, "bytes": 1572864 }, "scales": { "shard": 2, "offset": 63336448, "bytes": 49152 } }, "blocks.16.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 63385600, "bytes": 2048 } }, "blocks.16.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 2, "offset": 63387648, "bytes": 1048576 }, "scales": { "shard": 2, "offset": 64436224, "bytes": 32768 } }, "blocks.16.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 64468992, "bytes": 2048 } }, "blocks.16.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 2, "offset": 64471040, "bytes": 1048576 }, "scales": { "shard": 2, "offset": 65519616, "bytes": 32768 } }, "blocks.16.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 2, "offset": 65552384, "bytes": 8192 } }, "blocks.16.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 3, "offset": 0, "bytes": 4194304 }, "scales": { "shard": 3, "offset": 4194304, "bytes": 131072 } }, "blocks.16.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 4325376, "bytes": 2048 } }, "blocks.16.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 3, "offset": 4327424, "bytes": 4194304 }, "scales": { "shard": 3, "offset": 8521728, "bytes": 131072 } }, "blocks.16.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 8652800, "bytes": 2048 } }, "blocks.16.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 8654848, "bytes": 2048 } }, "blocks.16.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 8656896, "bytes": 2048 } }, "blocks.16.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 8658944, "bytes": 2048 } }, "blocks.16.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 8660992, "bytes": 2048 } }, "blocks.16.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 3, "offset": 8663040, "bytes": 1048576 }, "scales": { "shard": 3, "offset": 9711616, "bytes": 32768 } }, "blocks.16.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 9744384, "bytes": 6144 } }, "blocks.16.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 3, "offset": 9750528, "bytes": 3145728 }, "scales": { "shard": 3, "offset": 12896256, "bytes": 98304 } }, "blocks.17.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 12994560, "bytes": 12288 } }, "blocks.17.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 3, "offset": 13006848, "bytes": 6291456 }, "scales": { "shard": 3, "offset": 19298304, "bytes": 196608 } }, "blocks.17.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 19494912, "bytes": 4096 } }, "blocks.17.cross_attn.to_kv.weight": { "shape": [ 2048, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 768, "padded_rows": 2048, "padded_columns": 768, "data": { "shard": 3, "offset": 19499008, "bytes": 1572864 }, "scales": { "shard": 3, "offset": 21071872, "bytes": 49152 } }, "blocks.17.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 21121024, "bytes": 2048 } }, "blocks.17.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 3, "offset": 21123072, "bytes": 1048576 }, "scales": { "shard": 3, "offset": 22171648, "bytes": 32768 } }, "blocks.17.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 22204416, "bytes": 2048 } }, "blocks.17.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 3, "offset": 22206464, "bytes": 1048576 }, "scales": { "shard": 3, "offset": 23255040, "bytes": 32768 } }, "blocks.17.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 23287808, "bytes": 8192 } }, "blocks.17.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 3, "offset": 23296000, "bytes": 4194304 }, "scales": { "shard": 3, "offset": 27490304, "bytes": 131072 } }, "blocks.17.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 27621376, "bytes": 2048 } }, "blocks.17.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 3, "offset": 27623424, "bytes": 4194304 }, "scales": { "shard": 3, "offset": 31817728, "bytes": 131072 } }, "blocks.17.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 31948800, "bytes": 2048 } }, "blocks.17.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 31950848, "bytes": 2048 } }, "blocks.17.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 31952896, "bytes": 2048 } }, "blocks.17.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 31954944, "bytes": 2048 } }, "blocks.17.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 31956992, "bytes": 2048 } }, "blocks.17.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 3, "offset": 31959040, "bytes": 1048576 }, "scales": { "shard": 3, "offset": 33007616, "bytes": 32768 } }, "blocks.17.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 33040384, "bytes": 6144 } }, "blocks.17.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 3, "offset": 33046528, "bytes": 3145728 }, "scales": { "shard": 3, "offset": 36192256, "bytes": 98304 } }, "blocks.18.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 36290560, "bytes": 12288 } }, "blocks.18.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 3, "offset": 36302848, "bytes": 6291456 }, "scales": { "shard": 3, "offset": 42594304, "bytes": 196608 } }, "blocks.18.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 42790912, "bytes": 4096 } }, "blocks.18.cross_attn.to_kv.weight": { "shape": [ 2048, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 768, "padded_rows": 2048, "padded_columns": 768, "data": { "shard": 3, "offset": 42795008, "bytes": 1572864 }, "scales": { "shard": 3, "offset": 44367872, "bytes": 49152 } }, "blocks.18.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 44417024, "bytes": 2048 } }, "blocks.18.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 3, "offset": 44419072, "bytes": 1048576 }, "scales": { "shard": 3, "offset": 45467648, "bytes": 32768 } }, "blocks.18.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 45500416, "bytes": 2048 } }, "blocks.18.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 3, "offset": 45502464, "bytes": 1048576 }, "scales": { "shard": 3, "offset": 46551040, "bytes": 32768 } }, "blocks.18.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 46583808, "bytes": 8192 } }, "blocks.18.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 3, "offset": 46592000, "bytes": 4194304 }, "scales": { "shard": 3, "offset": 50786304, "bytes": 131072 } }, "blocks.18.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 50917376, "bytes": 2048 } }, "blocks.18.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 3, "offset": 50919424, "bytes": 4194304 }, "scales": { "shard": 3, "offset": 55113728, "bytes": 131072 } }, "blocks.18.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 55244800, "bytes": 2048 } }, "blocks.18.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 55246848, "bytes": 2048 } }, "blocks.18.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 55248896, "bytes": 2048 } }, "blocks.18.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 55250944, "bytes": 2048 } }, "blocks.18.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 55252992, "bytes": 2048 } }, "blocks.18.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 3, "offset": 55255040, "bytes": 1048576 }, "scales": { "shard": 3, "offset": 56303616, "bytes": 32768 } }, "blocks.18.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 56336384, "bytes": 6144 } }, "blocks.18.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 3, "offset": 56342528, "bytes": 3145728 }, "scales": { "shard": 3, "offset": 59488256, "bytes": 98304 } }, "blocks.19.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 59586560, "bytes": 12288 } }, "blocks.19.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 3, "offset": 59598848, "bytes": 6291456 }, "scales": { "shard": 3, "offset": 65890304, "bytes": 196608 } }, "blocks.19.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 3, "offset": 66086912, "bytes": 4096 } }, "blocks.19.cross_attn.to_kv.weight": { "shape": [ 2048, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 768, "padded_rows": 2048, "padded_columns": 768, "data": { "shard": 4, "offset": 0, "bytes": 1572864 }, "scales": { "shard": 4, "offset": 1572864, "bytes": 49152 } }, "blocks.19.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 1622016, "bytes": 2048 } }, "blocks.19.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 4, "offset": 1624064, "bytes": 1048576 }, "scales": { "shard": 4, "offset": 2672640, "bytes": 32768 } }, "blocks.19.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 2705408, "bytes": 2048 } }, "blocks.19.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 4, "offset": 2707456, "bytes": 1048576 }, "scales": { "shard": 4, "offset": 3756032, "bytes": 32768 } }, "blocks.19.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 3788800, "bytes": 8192 } }, "blocks.19.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 4, "offset": 3796992, "bytes": 4194304 }, "scales": { "shard": 4, "offset": 7991296, "bytes": 131072 } }, "blocks.19.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 8122368, "bytes": 2048 } }, "blocks.19.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 4, "offset": 8124416, "bytes": 4194304 }, "scales": { "shard": 4, "offset": 12318720, "bytes": 131072 } }, "blocks.19.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 12449792, "bytes": 2048 } }, "blocks.19.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 12451840, "bytes": 2048 } }, "blocks.19.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 12453888, "bytes": 2048 } }, "blocks.19.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 12455936, "bytes": 2048 } }, "blocks.19.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 12457984, "bytes": 2048 } }, "blocks.19.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 4, "offset": 12460032, "bytes": 1048576 }, "scales": { "shard": 4, "offset": 13508608, "bytes": 32768 } }, "blocks.19.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 13541376, "bytes": 6144 } }, "blocks.19.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 4, "offset": 13547520, "bytes": 3145728 }, "scales": { "shard": 4, "offset": 16693248, "bytes": 98304 } }, "blocks.2.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 16791552, "bytes": 12288 } }, "blocks.2.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 4, "offset": 16803840, "bytes": 6291456 }, "scales": { "shard": 4, "offset": 23095296, "bytes": 196608 } }, "blocks.2.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 23291904, "bytes": 4096 } }, "blocks.2.cross_attn.to_kv.weight": { "shape": [ 2048, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 768, "padded_rows": 2048, "padded_columns": 768, "data": { "shard": 4, "offset": 23296000, "bytes": 1572864 }, "scales": { "shard": 4, "offset": 24868864, "bytes": 49152 } }, "blocks.2.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 24918016, "bytes": 2048 } }, "blocks.2.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 4, "offset": 24920064, "bytes": 1048576 }, "scales": { "shard": 4, "offset": 25968640, "bytes": 32768 } }, "blocks.2.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 26001408, "bytes": 2048 } }, "blocks.2.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 4, "offset": 26003456, "bytes": 1048576 }, "scales": { "shard": 4, "offset": 27052032, "bytes": 32768 } }, "blocks.2.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 27084800, "bytes": 8192 } }, "blocks.2.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 4, "offset": 27092992, "bytes": 4194304 }, "scales": { "shard": 4, "offset": 31287296, "bytes": 131072 } }, "blocks.2.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 31418368, "bytes": 2048 } }, "blocks.2.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 4, "offset": 31420416, "bytes": 4194304 }, "scales": { "shard": 4, "offset": 35614720, "bytes": 131072 } }, "blocks.2.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 35745792, "bytes": 2048 } }, "blocks.2.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 35747840, "bytes": 2048 } }, "blocks.2.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 35749888, "bytes": 2048 } }, "blocks.2.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 35751936, "bytes": 2048 } }, "blocks.2.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 35753984, "bytes": 2048 } }, "blocks.2.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 4, "offset": 35756032, "bytes": 1048576 }, "scales": { "shard": 4, "offset": 36804608, "bytes": 32768 } }, "blocks.2.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 36837376, "bytes": 6144 } }, "blocks.2.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 4, "offset": 36843520, "bytes": 3145728 }, "scales": { "shard": 4, "offset": 39989248, "bytes": 98304 } }, "blocks.20.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 40087552, "bytes": 12288 } }, "blocks.20.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 4, "offset": 40099840, "bytes": 6291456 }, "scales": { "shard": 4, "offset": 46391296, "bytes": 196608 } }, "blocks.20.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 46587904, "bytes": 4096 } }, "blocks.20.cross_attn.to_kv.weight": { "shape": [ 2048, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 768, "padded_rows": 2048, "padded_columns": 768, "data": { "shard": 4, "offset": 46592000, "bytes": 1572864 }, "scales": { "shard": 4, "offset": 48164864, "bytes": 49152 } }, "blocks.20.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 48214016, "bytes": 2048 } }, "blocks.20.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 4, "offset": 48216064, "bytes": 1048576 }, "scales": { "shard": 4, "offset": 49264640, "bytes": 32768 } }, "blocks.20.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 49297408, "bytes": 2048 } }, "blocks.20.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 4, "offset": 49299456, "bytes": 1048576 }, "scales": { "shard": 4, "offset": 50348032, "bytes": 32768 } }, "blocks.20.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 50380800, "bytes": 8192 } }, "blocks.20.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 4, "offset": 50388992, "bytes": 4194304 }, "scales": { "shard": 4, "offset": 54583296, "bytes": 131072 } }, "blocks.20.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 54714368, "bytes": 2048 } }, "blocks.20.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 4, "offset": 54716416, "bytes": 4194304 }, "scales": { "shard": 4, "offset": 58910720, "bytes": 131072 } }, "blocks.20.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 59041792, "bytes": 2048 } }, "blocks.20.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 59043840, "bytes": 2048 } }, "blocks.20.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 59045888, "bytes": 2048 } }, "blocks.20.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 59047936, "bytes": 2048 } }, "blocks.20.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 59049984, "bytes": 2048 } }, "blocks.20.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 4, "offset": 59052032, "bytes": 1048576 }, "scales": { "shard": 4, "offset": 60100608, "bytes": 32768 } }, "blocks.20.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 60133376, "bytes": 6144 } }, "blocks.20.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 4, "offset": 60139520, "bytes": 3145728 }, "scales": { "shard": 4, "offset": 63285248, "bytes": 98304 } }, "blocks.21.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 4, "offset": 63383552, "bytes": 12288 } }, "blocks.21.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 5, "offset": 0, "bytes": 6291456 }, "scales": { "shard": 5, "offset": 6291456, "bytes": 196608 } }, "blocks.21.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 6488064, "bytes": 4096 } }, "blocks.21.cross_attn.to_kv.weight": { "shape": [ 2048, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 768, "padded_rows": 2048, "padded_columns": 768, "data": { "shard": 5, "offset": 6492160, "bytes": 1572864 }, "scales": { "shard": 5, "offset": 8065024, "bytes": 49152 } }, "blocks.21.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 8114176, "bytes": 2048 } }, "blocks.21.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 5, "offset": 8116224, "bytes": 1048576 }, "scales": { "shard": 5, "offset": 9164800, "bytes": 32768 } }, "blocks.21.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 9197568, "bytes": 2048 } }, "blocks.21.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 5, "offset": 9199616, "bytes": 1048576 }, "scales": { "shard": 5, "offset": 10248192, "bytes": 32768 } }, "blocks.21.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 10280960, "bytes": 8192 } }, "blocks.21.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 5, "offset": 10289152, "bytes": 4194304 }, "scales": { "shard": 5, "offset": 14483456, "bytes": 131072 } }, "blocks.21.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 14614528, "bytes": 2048 } }, "blocks.21.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 5, "offset": 14616576, "bytes": 4194304 }, "scales": { "shard": 5, "offset": 18810880, "bytes": 131072 } }, "blocks.21.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 18941952, "bytes": 2048 } }, "blocks.21.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 18944000, "bytes": 2048 } }, "blocks.21.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 18946048, "bytes": 2048 } }, "blocks.21.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 18948096, "bytes": 2048 } }, "blocks.21.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 18950144, "bytes": 2048 } }, "blocks.21.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 5, "offset": 18952192, "bytes": 1048576 }, "scales": { "shard": 5, "offset": 20000768, "bytes": 32768 } }, "blocks.21.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 20033536, "bytes": 6144 } }, "blocks.21.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 5, "offset": 20039680, "bytes": 3145728 }, "scales": { "shard": 5, "offset": 23185408, "bytes": 98304 } }, "blocks.22.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 23283712, "bytes": 12288 } }, "blocks.22.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 5, "offset": 23296000, "bytes": 6291456 }, "scales": { "shard": 5, "offset": 29587456, "bytes": 196608 } }, "blocks.22.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 29784064, "bytes": 4096 } }, "blocks.22.cross_attn.to_kv.weight": { "shape": [ 2048, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 768, "padded_rows": 2048, "padded_columns": 768, "data": { "shard": 5, "offset": 29788160, "bytes": 1572864 }, "scales": { "shard": 5, "offset": 31361024, "bytes": 49152 } }, "blocks.22.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 31410176, "bytes": 2048 } }, "blocks.22.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 5, "offset": 31412224, "bytes": 1048576 }, "scales": { "shard": 5, "offset": 32460800, "bytes": 32768 } }, "blocks.22.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 32493568, "bytes": 2048 } }, "blocks.22.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 5, "offset": 32495616, "bytes": 1048576 }, "scales": { "shard": 5, "offset": 33544192, "bytes": 32768 } }, "blocks.22.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 33576960, "bytes": 8192 } }, "blocks.22.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 5, "offset": 33585152, "bytes": 4194304 }, "scales": { "shard": 5, "offset": 37779456, "bytes": 131072 } }, "blocks.22.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 37910528, "bytes": 2048 } }, "blocks.22.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 5, "offset": 37912576, "bytes": 4194304 }, "scales": { "shard": 5, "offset": 42106880, "bytes": 131072 } }, "blocks.22.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 42237952, "bytes": 2048 } }, "blocks.22.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 42240000, "bytes": 2048 } }, "blocks.22.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 42242048, "bytes": 2048 } }, "blocks.22.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 42244096, "bytes": 2048 } }, "blocks.22.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 42246144, "bytes": 2048 } }, "blocks.22.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 5, "offset": 42248192, "bytes": 1048576 }, "scales": { "shard": 5, "offset": 43296768, "bytes": 32768 } }, "blocks.22.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 43329536, "bytes": 6144 } }, "blocks.22.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 5, "offset": 43335680, "bytes": 3145728 }, "scales": { "shard": 5, "offset": 46481408, "bytes": 98304 } }, "blocks.23.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 46579712, "bytes": 12288 } }, "blocks.23.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 5, "offset": 46592000, "bytes": 6291456 }, "scales": { "shard": 5, "offset": 52883456, "bytes": 196608 } }, "blocks.23.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 53080064, "bytes": 4096 } }, "blocks.23.cross_attn.to_kv.weight": { "shape": [ 2048, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 768, "padded_rows": 2048, "padded_columns": 768, "data": { "shard": 5, "offset": 53084160, "bytes": 1572864 }, "scales": { "shard": 5, "offset": 54657024, "bytes": 49152 } }, "blocks.23.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 54706176, "bytes": 2048 } }, "blocks.23.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 5, "offset": 54708224, "bytes": 1048576 }, "scales": { "shard": 5, "offset": 55756800, "bytes": 32768 } }, "blocks.23.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 55789568, "bytes": 2048 } }, "blocks.23.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 5, "offset": 55791616, "bytes": 1048576 }, "scales": { "shard": 5, "offset": 56840192, "bytes": 32768 } }, "blocks.23.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 56872960, "bytes": 8192 } }, "blocks.23.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 5, "offset": 56881152, "bytes": 4194304 }, "scales": { "shard": 5, "offset": 61075456, "bytes": 131072 } }, "blocks.23.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 61206528, "bytes": 2048 } }, "blocks.23.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 5, "offset": 61208576, "bytes": 4194304 }, "scales": { "shard": 5, "offset": 65402880, "bytes": 131072 } }, "blocks.23.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 65533952, "bytes": 2048 } }, "blocks.23.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 65536000, "bytes": 2048 } }, "blocks.23.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 65538048, "bytes": 2048 } }, "blocks.23.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 65540096, "bytes": 2048 } }, "blocks.23.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 65542144, "bytes": 2048 } }, "blocks.23.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 5, "offset": 65544192, "bytes": 1048576 }, "scales": { "shard": 5, "offset": 66592768, "bytes": 32768 } }, "blocks.23.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 5, "offset": 66625536, "bytes": 6144 } }, "blocks.23.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 6, "offset": 0, "bytes": 3145728 }, "scales": { "shard": 6, "offset": 3145728, "bytes": 98304 } }, "blocks.3.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 3244032, "bytes": 12288 } }, "blocks.3.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 6, "offset": 3256320, "bytes": 6291456 }, "scales": { "shard": 6, "offset": 9547776, "bytes": 196608 } }, "blocks.3.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 9744384, "bytes": 4096 } }, "blocks.3.cross_attn.to_kv.weight": { "shape": [ 2048, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 768, "padded_rows": 2048, "padded_columns": 768, "data": { "shard": 6, "offset": 9748480, "bytes": 1572864 }, "scales": { "shard": 6, "offset": 11321344, "bytes": 49152 } }, "blocks.3.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 11370496, "bytes": 2048 } }, "blocks.3.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 6, "offset": 11372544, "bytes": 1048576 }, "scales": { "shard": 6, "offset": 12421120, "bytes": 32768 } }, "blocks.3.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 12453888, "bytes": 2048 } }, "blocks.3.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 6, "offset": 12455936, "bytes": 1048576 }, "scales": { "shard": 6, "offset": 13504512, "bytes": 32768 } }, "blocks.3.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 13537280, "bytes": 8192 } }, "blocks.3.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 6, "offset": 13545472, "bytes": 4194304 }, "scales": { "shard": 6, "offset": 17739776, "bytes": 131072 } }, "blocks.3.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 17870848, "bytes": 2048 } }, "blocks.3.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 6, "offset": 17872896, "bytes": 4194304 }, "scales": { "shard": 6, "offset": 22067200, "bytes": 131072 } }, "blocks.3.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 22198272, "bytes": 2048 } }, "blocks.3.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 22200320, "bytes": 2048 } }, "blocks.3.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 22202368, "bytes": 2048 } }, "blocks.3.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 22204416, "bytes": 2048 } }, "blocks.3.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 22206464, "bytes": 2048 } }, "blocks.3.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 6, "offset": 22208512, "bytes": 1048576 }, "scales": { "shard": 6, "offset": 23257088, "bytes": 32768 } }, "blocks.3.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 23289856, "bytes": 6144 } }, "blocks.3.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 6, "offset": 23296000, "bytes": 3145728 }, "scales": { "shard": 6, "offset": 26441728, "bytes": 98304 } }, "blocks.4.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 26540032, "bytes": 12288 } }, "blocks.4.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 6, "offset": 26552320, "bytes": 6291456 }, "scales": { "shard": 6, "offset": 32843776, "bytes": 196608 } }, "blocks.4.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 33040384, "bytes": 4096 } }, "blocks.4.cross_attn.to_kv.weight": { "shape": [ 2048, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 768, "padded_rows": 2048, "padded_columns": 768, "data": { "shard": 6, "offset": 33044480, "bytes": 1572864 }, "scales": { "shard": 6, "offset": 34617344, "bytes": 49152 } }, "blocks.4.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 34666496, "bytes": 2048 } }, "blocks.4.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 6, "offset": 34668544, "bytes": 1048576 }, "scales": { "shard": 6, "offset": 35717120, "bytes": 32768 } }, "blocks.4.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 35749888, "bytes": 2048 } }, "blocks.4.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 6, "offset": 35751936, "bytes": 1048576 }, "scales": { "shard": 6, "offset": 36800512, "bytes": 32768 } }, "blocks.4.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 36833280, "bytes": 8192 } }, "blocks.4.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 6, "offset": 36841472, "bytes": 4194304 }, "scales": { "shard": 6, "offset": 41035776, "bytes": 131072 } }, "blocks.4.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 41166848, "bytes": 2048 } }, "blocks.4.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 6, "offset": 41168896, "bytes": 4194304 }, "scales": { "shard": 6, "offset": 45363200, "bytes": 131072 } }, "blocks.4.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 45494272, "bytes": 2048 } }, "blocks.4.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 45496320, "bytes": 2048 } }, "blocks.4.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 45498368, "bytes": 2048 } }, "blocks.4.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 45500416, "bytes": 2048 } }, "blocks.4.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 45502464, "bytes": 2048 } }, "blocks.4.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 6, "offset": 45504512, "bytes": 1048576 }, "scales": { "shard": 6, "offset": 46553088, "bytes": 32768 } }, "blocks.4.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 46585856, "bytes": 6144 } }, "blocks.4.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 6, "offset": 46592000, "bytes": 3145728 }, "scales": { "shard": 6, "offset": 49737728, "bytes": 98304 } }, "blocks.5.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 49836032, "bytes": 12288 } }, "blocks.5.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 6, "offset": 49848320, "bytes": 6291456 }, "scales": { "shard": 6, "offset": 56139776, "bytes": 196608 } }, "blocks.5.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 56336384, "bytes": 4096 } }, "blocks.5.cross_attn.to_kv.weight": { "shape": [ 2048, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 768, "padded_rows": 2048, "padded_columns": 768, "data": { "shard": 6, "offset": 56340480, "bytes": 1572864 }, "scales": { "shard": 6, "offset": 57913344, "bytes": 49152 } }, "blocks.5.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 57962496, "bytes": 2048 } }, "blocks.5.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 6, "offset": 57964544, "bytes": 1048576 }, "scales": { "shard": 6, "offset": 59013120, "bytes": 32768 } }, "blocks.5.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 59045888, "bytes": 2048 } }, "blocks.5.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 6, "offset": 59047936, "bytes": 1048576 }, "scales": { "shard": 6, "offset": 60096512, "bytes": 32768 } }, "blocks.5.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 60129280, "bytes": 8192 } }, "blocks.5.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 6, "offset": 60137472, "bytes": 4194304 }, "scales": { "shard": 6, "offset": 64331776, "bytes": 131072 } }, "blocks.5.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 6, "offset": 64462848, "bytes": 2048 } }, "blocks.5.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 7, "offset": 0, "bytes": 4194304 }, "scales": { "shard": 7, "offset": 4194304, "bytes": 131072 } }, "blocks.5.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 4325376, "bytes": 2048 } }, "blocks.5.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 4327424, "bytes": 2048 } }, "blocks.5.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 4329472, "bytes": 2048 } }, "blocks.5.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 4331520, "bytes": 2048 } }, "blocks.5.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 4333568, "bytes": 2048 } }, "blocks.5.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 7, "offset": 4335616, "bytes": 1048576 }, "scales": { "shard": 7, "offset": 5384192, "bytes": 32768 } }, "blocks.5.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 5416960, "bytes": 6144 } }, "blocks.5.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 7, "offset": 5423104, "bytes": 3145728 }, "scales": { "shard": 7, "offset": 8568832, "bytes": 98304 } }, "blocks.6.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 8667136, "bytes": 12288 } }, "blocks.6.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 7, "offset": 8679424, "bytes": 6291456 }, "scales": { "shard": 7, "offset": 14970880, "bytes": 196608 } }, "blocks.6.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 15167488, "bytes": 4096 } }, "blocks.6.cross_attn.to_kv.weight": { "shape": [ 2048, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 768, "padded_rows": 2048, "padded_columns": 768, "data": { "shard": 7, "offset": 15171584, "bytes": 1572864 }, "scales": { "shard": 7, "offset": 16744448, "bytes": 49152 } }, "blocks.6.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 16793600, "bytes": 2048 } }, "blocks.6.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 7, "offset": 16795648, "bytes": 1048576 }, "scales": { "shard": 7, "offset": 17844224, "bytes": 32768 } }, "blocks.6.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 17876992, "bytes": 2048 } }, "blocks.6.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 7, "offset": 17879040, "bytes": 1048576 }, "scales": { "shard": 7, "offset": 18927616, "bytes": 32768 } }, "blocks.6.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 18960384, "bytes": 8192 } }, "blocks.6.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 7, "offset": 18968576, "bytes": 4194304 }, "scales": { "shard": 7, "offset": 23162880, "bytes": 131072 } }, "blocks.6.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 23293952, "bytes": 2048 } }, "blocks.6.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 7, "offset": 23296000, "bytes": 4194304 }, "scales": { "shard": 7, "offset": 27490304, "bytes": 131072 } }, "blocks.6.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 27621376, "bytes": 2048 } }, "blocks.6.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 27623424, "bytes": 2048 } }, "blocks.6.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 27625472, "bytes": 2048 } }, "blocks.6.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 27627520, "bytes": 2048 } }, "blocks.6.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 27629568, "bytes": 2048 } }, "blocks.6.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 7, "offset": 27631616, "bytes": 1048576 }, "scales": { "shard": 7, "offset": 28680192, "bytes": 32768 } }, "blocks.6.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 28712960, "bytes": 6144 } }, "blocks.6.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 7, "offset": 28719104, "bytes": 3145728 }, "scales": { "shard": 7, "offset": 31864832, "bytes": 98304 } }, "blocks.7.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 31963136, "bytes": 12288 } }, "blocks.7.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 7, "offset": 31975424, "bytes": 6291456 }, "scales": { "shard": 7, "offset": 38266880, "bytes": 196608 } }, "blocks.7.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 38463488, "bytes": 4096 } }, "blocks.7.cross_attn.to_kv.weight": { "shape": [ 2048, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 768, "padded_rows": 2048, "padded_columns": 768, "data": { "shard": 7, "offset": 38467584, "bytes": 1572864 }, "scales": { "shard": 7, "offset": 40040448, "bytes": 49152 } }, "blocks.7.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 40089600, "bytes": 2048 } }, "blocks.7.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 7, "offset": 40091648, "bytes": 1048576 }, "scales": { "shard": 7, "offset": 41140224, "bytes": 32768 } }, "blocks.7.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 41172992, "bytes": 2048 } }, "blocks.7.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 7, "offset": 41175040, "bytes": 1048576 }, "scales": { "shard": 7, "offset": 42223616, "bytes": 32768 } }, "blocks.7.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 42256384, "bytes": 8192 } }, "blocks.7.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 7, "offset": 42264576, "bytes": 4194304 }, "scales": { "shard": 7, "offset": 46458880, "bytes": 131072 } }, "blocks.7.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 46589952, "bytes": 2048 } }, "blocks.7.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 7, "offset": 46592000, "bytes": 4194304 }, "scales": { "shard": 7, "offset": 50786304, "bytes": 131072 } }, "blocks.7.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 50917376, "bytes": 2048 } }, "blocks.7.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 50919424, "bytes": 2048 } }, "blocks.7.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 50921472, "bytes": 2048 } }, "blocks.7.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 50923520, "bytes": 2048 } }, "blocks.7.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 50925568, "bytes": 2048 } }, "blocks.7.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 7, "offset": 50927616, "bytes": 1048576 }, "scales": { "shard": 7, "offset": 51976192, "bytes": 32768 } }, "blocks.7.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 52008960, "bytes": 6144 } }, "blocks.7.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 7, "offset": 52015104, "bytes": 3145728 }, "scales": { "shard": 7, "offset": 55160832, "bytes": 98304 } }, "blocks.8.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 55259136, "bytes": 12288 } }, "blocks.8.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 7, "offset": 55271424, "bytes": 6291456 }, "scales": { "shard": 7, "offset": 61562880, "bytes": 196608 } }, "blocks.8.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 61759488, "bytes": 4096 } }, "blocks.8.cross_attn.to_kv.weight": { "shape": [ 2048, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 768, "padded_rows": 2048, "padded_columns": 768, "data": { "shard": 7, "offset": 61763584, "bytes": 1572864 }, "scales": { "shard": 7, "offset": 63336448, "bytes": 49152 } }, "blocks.8.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 63385600, "bytes": 2048 } }, "blocks.8.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 7, "offset": 63387648, "bytes": 1048576 }, "scales": { "shard": 7, "offset": 64436224, "bytes": 32768 } }, "blocks.8.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 64468992, "bytes": 2048 } }, "blocks.8.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 7, "offset": 64471040, "bytes": 1048576 }, "scales": { "shard": 7, "offset": 65519616, "bytes": 32768 } }, "blocks.8.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 7, "offset": 65552384, "bytes": 8192 } }, "blocks.8.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 8, "offset": 0, "bytes": 4194304 }, "scales": { "shard": 8, "offset": 4194304, "bytes": 131072 } }, "blocks.8.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 4325376, "bytes": 2048 } }, "blocks.8.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 8, "offset": 4327424, "bytes": 4194304 }, "scales": { "shard": 8, "offset": 8521728, "bytes": 131072 } }, "blocks.8.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 8652800, "bytes": 2048 } }, "blocks.8.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 8654848, "bytes": 2048 } }, "blocks.8.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 8656896, "bytes": 2048 } }, "blocks.8.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 8658944, "bytes": 2048 } }, "blocks.8.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 8660992, "bytes": 2048 } }, "blocks.8.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 8, "offset": 8663040, "bytes": 1048576 }, "scales": { "shard": 8, "offset": 9711616, "bytes": 32768 } }, "blocks.8.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 9744384, "bytes": 6144 } }, "blocks.8.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 8, "offset": 9750528, "bytes": 3145728 }, "scales": { "shard": 8, "offset": 12896256, "bytes": 98304 } }, "blocks.9.adaLN_modulation.1.bias": { "shape": [ 6144 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 12994560, "bytes": 12288 } }, "blocks.9.adaLN_modulation.1.weight": { "shape": [ 6144, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 6144, "columns": 1024, "padded_rows": 6144, "padded_columns": 1024, "data": { "shard": 8, "offset": 13006848, "bytes": 6291456 }, "scales": { "shard": 8, "offset": 19298304, "bytes": 196608 } }, "blocks.9.cross_attn.to_kv.bias": { "shape": [ 2048 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 19494912, "bytes": 4096 } }, "blocks.9.cross_attn.to_kv.weight": { "shape": [ 2048, 768 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 2048, "columns": 768, "padded_rows": 2048, "padded_columns": 768, "data": { "shard": 8, "offset": 19499008, "bytes": 1572864 }, "scales": { "shard": 8, "offset": 21071872, "bytes": 49152 } }, "blocks.9.cross_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 21121024, "bytes": 2048 } }, "blocks.9.cross_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 8, "offset": 21123072, "bytes": 1048576 }, "scales": { "shard": 8, "offset": 22171648, "bytes": 32768 } }, "blocks.9.cross_attn.to_q.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 22204416, "bytes": 2048 } }, "blocks.9.cross_attn.to_q.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 8, "offset": 22206464, "bytes": 1048576 }, "scales": { "shard": 8, "offset": 23255040, "bytes": 32768 } }, "blocks.9.mlp.mlp.0.bias": { "shape": [ 4096 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 23287808, "bytes": 8192 } }, "blocks.9.mlp.mlp.0.weight": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 8, "offset": 23296000, "bytes": 4194304 }, "scales": { "shard": 8, "offset": 27490304, "bytes": 131072 } }, "blocks.9.mlp.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 27621376, "bytes": 2048 } }, "blocks.9.mlp.mlp.2.weight": { "shape": [ 1024, 4096 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 4096, "padded_rows": 1024, "padded_columns": 4096, "data": { "shard": 8, "offset": 27623424, "bytes": 4194304 }, "scales": { "shard": 8, "offset": 31817728, "bytes": 131072 } }, "blocks.9.norm2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 31948800, "bytes": 2048 } }, "blocks.9.norm2.weight": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 31950848, "bytes": 2048 } }, "blocks.9.self_attn.k_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 31952896, "bytes": 2048 } }, "blocks.9.self_attn.q_rms_norm.gamma": { "shape": [ 16, 64 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 31954944, "bytes": 2048 } }, "blocks.9.self_attn.to_out.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 31956992, "bytes": 2048 } }, "blocks.9.self_attn.to_out.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 8, "offset": 31959040, "bytes": 1048576 }, "scales": { "shard": 8, "offset": 33007616, "bytes": 32768 } }, "blocks.9.self_attn.to_qkv.bias": { "shape": [ 3072 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 33040384, "bytes": 6144 } }, "blocks.9.self_attn.to_qkv.weight": { "shape": [ 3072, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 3072, "columns": 1024, "padded_rows": 3072, "padded_columns": 1024, "data": { "shard": 8, "offset": 33046528, "bytes": 3145728 }, "scales": { "shard": 8, "offset": 36192256, "bytes": 98304 } }, "input_layer.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 36290560, "bytes": 2048 } }, "input_layer.weight": { "shape": [ 1024, 8 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 8, "padded_rows": 1024, "padded_columns": 64, "data": { "shard": 8, "offset": 36292608, "bytes": 65536 }, "scales": { "shard": 8, "offset": 36358144, "bytes": 2048 } }, "out_layer.bias": { "shape": [ 8 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 36360192, "bytes": 16 } }, "out_layer.weight": { "shape": [ 8, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 8, "columns": 1024, "padded_rows": 16, "padded_columns": 1024, "data": { "shard": 8, "offset": 36360448, "bytes": 16384 }, "scales": { "shard": 8, "offset": 36376832, "bytes": 512 } }, "pos_emb": { "shape": [ 4096, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 4096, "columns": 1024, "padded_rows": 4096, "padded_columns": 1024, "data": { "shard": 8, "offset": 36377344, "bytes": 4194304 }, "scales": { "shard": 8, "offset": 40571648, "bytes": 131072 } }, "t_embedder.mlp.0.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 40702720, "bytes": 2048 } }, "t_embedder.mlp.0.weight": { "shape": [ 1024, 256 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 256, "padded_rows": 1024, "padded_columns": 256, "data": { "shard": 8, "offset": 40704768, "bytes": 262144 }, "scales": { "shard": 8, "offset": 40966912, "bytes": 8192 } }, "t_embedder.mlp.2.bias": { "shape": [ 1024 ], "encoding": "raw", "dtype": "f16", "data": { "shard": 8, "offset": 40975104, "bytes": 2048 } }, "t_embedder.mlp.2.weight": { "shape": [ 1024, 1024 ], "encoding": "symmetric-quantized", "bits": 8, "group_size": 64, "output_tile": 16, "rows": 1024, "columns": 1024, "padded_rows": 1024, "padded_columns": 1024, "data": { "shard": 8, "offset": 40977152, "bytes": 1048576 }, "scales": { "shard": 8, "offset": 42025728, "bytes": 32768 } } } }