| { |
| "schema": "trellis-webgpu-pack", |
| "version": 1, |
| "stage": "sparseStructureFlow", |
| "upstreamRevision": "4aad9f4a110329a410974d7f41ce5333a9a1fc87", |
| "shards": [ |
| { |
| "file": "stage-000.bin", |
| "bytes": 66643968, |
| "sha256": "05087b26ad671d96adae17ad09d6ad30b97925440027c51f91e2bf511af93e77" |
| }, |
| { |
| "file": "stage-001.bin", |
| "bytes": 64464896, |
| "sha256": "2e2e70521febf08cd1869e606ed38a5d2f20d539d05e4188490f79c59a674e47" |
| }, |
| { |
| "file": "stage-002.bin", |
| "bytes": 65560576, |
| "sha256": "a54270c6ad310a93267260f07651ba7064b8477f1d58028a4f848089241c82bb" |
| }, |
| { |
| "file": "stage-003.bin", |
| "bytes": 66091008, |
| "sha256": "fb2c2fba4108b9eb0e099c207b19d0de99aed1eca3436d082282aab1483cb323" |
| }, |
| { |
| "file": "stage-004.bin", |
| "bytes": 63395840, |
| "sha256": "ba5d1e2802670282240691d2b5cab092ddc3a0d62f8beac1890f18f2bc812aee" |
| }, |
| { |
| "file": "stage-005.bin", |
| "bytes": 66631680, |
| "sha256": "4a64c79811c881cbde281b13ae11a3e8375902cead1126f07dd62e422da5de07" |
| }, |
| { |
| "file": "stage-006.bin", |
| "bytes": 64464896, |
| "sha256": "a5dc01ce0eae158edcb07fb7d63865f97c35c87a58ac0f565356126551672984" |
| }, |
| { |
| "file": "stage-007.bin", |
| "bytes": 65560576, |
| "sha256": "fbd8d6a918fca4718f50fe296ca69710b22826180790030bd735e040a4762a2b" |
| }, |
| { |
| "file": "stage-008.bin", |
| "bytes": 42058496, |
| "sha256": "126a3f3b2e704bad719342ad826ac774fc9b176b84eda7ec8a7e2234958b57c9" |
| } |
| ], |
| "tensors": { |
| "blocks.0.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 0, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.0.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 0, |
| "offset": 12288, |
| "bytes": 6291456 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 6303744, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.0.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 6500352, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.0.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 768, |
| "padded_rows": 2048, |
| "padded_columns": 768, |
| "data": { |
| "shard": 0, |
| "offset": 6504448, |
| "bytes": 1572864 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 8077312, |
| "bytes": 49152 |
| } |
| }, |
| "blocks.0.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 8126464, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.0.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 0, |
| "offset": 8128512, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 9177088, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.0.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 9209856, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.0.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 0, |
| "offset": 9211904, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 10260480, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.0.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 10293248, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.0.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 0, |
| "offset": 10301440, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 14495744, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.0.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 14626816, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.0.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "data": { |
| "shard": 0, |
| "offset": 14628864, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 18823168, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.0.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 18954240, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.0.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 18956288, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.0.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 18958336, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.0.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 18960384, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.0.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 18962432, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.0.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 0, |
| "offset": 18964480, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 20013056, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.0.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 20045824, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.0.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 0, |
| "offset": 20051968, |
| "bytes": 3145728 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 23197696, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.1.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 23296000, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.1.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 0, |
| "offset": 23308288, |
| "bytes": 6291456 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 29599744, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.1.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 29796352, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.1.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 768, |
| "padded_rows": 2048, |
| "padded_columns": 768, |
| "data": { |
| "shard": 0, |
| "offset": 29800448, |
| "bytes": 1572864 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 31373312, |
| "bytes": 49152 |
| } |
| }, |
| "blocks.1.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 31422464, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.1.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 0, |
| "offset": 31424512, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 32473088, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.1.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 32505856, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.1.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 0, |
| "offset": 32507904, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 33556480, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.1.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 33589248, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.1.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 0, |
| "offset": 33597440, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 37791744, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.1.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 37922816, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.1.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "data": { |
| "shard": 0, |
| "offset": 37924864, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 42119168, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.1.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 42250240, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.1.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 42252288, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.1.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 42254336, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.1.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 42256384, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.1.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 42258432, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.1.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 0, |
| "offset": 42260480, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 43309056, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.1.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 43341824, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.1.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 0, |
| "offset": 43347968, |
| "bytes": 3145728 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 46493696, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.10.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 46592000, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.10.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 0, |
| "offset": 46604288, |
| "bytes": 6291456 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 52895744, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.10.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 53092352, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.10.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 768, |
| "padded_rows": 2048, |
| "padded_columns": 768, |
| "data": { |
| "shard": 0, |
| "offset": 53096448, |
| "bytes": 1572864 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 54669312, |
| "bytes": 49152 |
| } |
| }, |
| "blocks.10.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 54718464, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.10.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 0, |
| "offset": 54720512, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 55769088, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.10.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 55801856, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.10.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 0, |
| "offset": 55803904, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 56852480, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.10.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 56885248, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.10.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 0, |
| "offset": 56893440, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 61087744, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.10.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 61218816, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.10.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "data": { |
| "shard": 0, |
| "offset": 61220864, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 65415168, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.10.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 65546240, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.10.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 65548288, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.10.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 65550336, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.10.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 65552384, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.10.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 65554432, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.10.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 0, |
| "offset": 65556480, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 0, |
| "offset": 66605056, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.10.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 0, |
| "offset": 66637824, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.10.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 1, |
| "offset": 0, |
| "bytes": 3145728 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 3145728, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.11.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 3244032, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.11.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 1, |
| "offset": 3256320, |
| "bytes": 6291456 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 9547776, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.11.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 9744384, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.11.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 768, |
| "padded_rows": 2048, |
| "padded_columns": 768, |
| "data": { |
| "shard": 1, |
| "offset": 9748480, |
| "bytes": 1572864 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 11321344, |
| "bytes": 49152 |
| } |
| }, |
| "blocks.11.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 11370496, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.11.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 1, |
| "offset": 11372544, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 12421120, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.11.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 12453888, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.11.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 1, |
| "offset": 12455936, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 13504512, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.11.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 13537280, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.11.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 1, |
| "offset": 13545472, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 17739776, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.11.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 17870848, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.11.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "data": { |
| "shard": 1, |
| "offset": 17872896, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 22067200, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.11.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 22198272, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.11.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 22200320, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.11.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 22202368, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.11.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 22204416, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.11.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 22206464, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.11.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 1, |
| "offset": 22208512, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 23257088, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.11.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 23289856, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.11.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 1, |
| "offset": 23296000, |
| "bytes": 3145728 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 26441728, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.12.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 26540032, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.12.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 1, |
| "offset": 26552320, |
| "bytes": 6291456 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 32843776, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.12.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 33040384, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.12.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 768, |
| "padded_rows": 2048, |
| "padded_columns": 768, |
| "data": { |
| "shard": 1, |
| "offset": 33044480, |
| "bytes": 1572864 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 34617344, |
| "bytes": 49152 |
| } |
| }, |
| "blocks.12.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 34666496, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.12.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 1, |
| "offset": 34668544, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 35717120, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.12.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 35749888, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.12.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 1, |
| "offset": 35751936, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 36800512, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.12.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 36833280, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.12.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 1, |
| "offset": 36841472, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 41035776, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.12.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 41166848, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.12.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "data": { |
| "shard": 1, |
| "offset": 41168896, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 45363200, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.12.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 45494272, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.12.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 45496320, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.12.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 45498368, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.12.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 45500416, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.12.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 45502464, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.12.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 1, |
| "offset": 45504512, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 46553088, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.12.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 46585856, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.12.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 1, |
| "offset": 46592000, |
| "bytes": 3145728 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 49737728, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.13.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 49836032, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.13.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 1, |
| "offset": 49848320, |
| "bytes": 6291456 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 56139776, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.13.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 56336384, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.13.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 768, |
| "padded_rows": 2048, |
| "padded_columns": 768, |
| "data": { |
| "shard": 1, |
| "offset": 56340480, |
| "bytes": 1572864 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 57913344, |
| "bytes": 49152 |
| } |
| }, |
| "blocks.13.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 57962496, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.13.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 1, |
| "offset": 57964544, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 59013120, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.13.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 59045888, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.13.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 1, |
| "offset": 59047936, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 60096512, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.13.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 60129280, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.13.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 1, |
| "offset": 60137472, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 1, |
| "offset": 64331776, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.13.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 1, |
| "offset": 64462848, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.13.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "data": { |
| "shard": 2, |
| "offset": 0, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 4194304, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.13.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 4325376, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.13.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 4327424, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.13.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 4329472, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.13.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 4331520, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.13.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 4333568, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.13.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 2, |
| "offset": 4335616, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 5384192, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.13.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 5416960, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.13.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 2, |
| "offset": 5423104, |
| "bytes": 3145728 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 8568832, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.14.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 8667136, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.14.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 2, |
| "offset": 8679424, |
| "bytes": 6291456 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 14970880, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.14.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 15167488, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.14.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 768, |
| "padded_rows": 2048, |
| "padded_columns": 768, |
| "data": { |
| "shard": 2, |
| "offset": 15171584, |
| "bytes": 1572864 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 16744448, |
| "bytes": 49152 |
| } |
| }, |
| "blocks.14.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 16793600, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.14.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 2, |
| "offset": 16795648, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 17844224, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.14.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 17876992, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.14.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 2, |
| "offset": 17879040, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 18927616, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.14.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 18960384, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.14.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 2, |
| "offset": 18968576, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 23162880, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.14.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 23293952, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.14.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "data": { |
| "shard": 2, |
| "offset": 23296000, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 27490304, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.14.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 27621376, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.14.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 27623424, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.14.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 27625472, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.14.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 27627520, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.14.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 27629568, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.14.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 2, |
| "offset": 27631616, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 28680192, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.14.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 28712960, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.14.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 2, |
| "offset": 28719104, |
| "bytes": 3145728 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 31864832, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.15.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 31963136, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.15.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 2, |
| "offset": 31975424, |
| "bytes": 6291456 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 38266880, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.15.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 38463488, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.15.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 768, |
| "padded_rows": 2048, |
| "padded_columns": 768, |
| "data": { |
| "shard": 2, |
| "offset": 38467584, |
| "bytes": 1572864 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 40040448, |
| "bytes": 49152 |
| } |
| }, |
| "blocks.15.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 40089600, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.15.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 2, |
| "offset": 40091648, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 41140224, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.15.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 41172992, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.15.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 2, |
| "offset": 41175040, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 42223616, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.15.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 42256384, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.15.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 2, |
| "offset": 42264576, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 46458880, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.15.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 46589952, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.15.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "data": { |
| "shard": 2, |
| "offset": 46592000, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 50786304, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.15.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 50917376, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.15.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 50919424, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.15.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 50921472, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.15.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 50923520, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.15.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 50925568, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.15.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 2, |
| "offset": 50927616, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 51976192, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.15.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 52008960, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.15.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 2, |
| "offset": 52015104, |
| "bytes": 3145728 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 55160832, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.16.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 55259136, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.16.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 2, |
| "offset": 55271424, |
| "bytes": 6291456 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 61562880, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.16.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 61759488, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.16.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 768, |
| "padded_rows": 2048, |
| "padded_columns": 768, |
| "data": { |
| "shard": 2, |
| "offset": 61763584, |
| "bytes": 1572864 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 63336448, |
| "bytes": 49152 |
| } |
| }, |
| "blocks.16.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 63385600, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.16.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 2, |
| "offset": 63387648, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 64436224, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.16.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 64468992, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.16.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 2, |
| "offset": 64471040, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 2, |
| "offset": 65519616, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.16.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 2, |
| "offset": 65552384, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.16.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 3, |
| "offset": 0, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 4194304, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.16.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 4325376, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.16.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "data": { |
| "shard": 3, |
| "offset": 4327424, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 8521728, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.16.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 8652800, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.16.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 8654848, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.16.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 8656896, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.16.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 8658944, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.16.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 8660992, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.16.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 3, |
| "offset": 8663040, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 9711616, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.16.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 9744384, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.16.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 3, |
| "offset": 9750528, |
| "bytes": 3145728 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 12896256, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.17.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 12994560, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.17.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 3, |
| "offset": 13006848, |
| "bytes": 6291456 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 19298304, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.17.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 19494912, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.17.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 768, |
| "padded_rows": 2048, |
| "padded_columns": 768, |
| "data": { |
| "shard": 3, |
| "offset": 19499008, |
| "bytes": 1572864 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 21071872, |
| "bytes": 49152 |
| } |
| }, |
| "blocks.17.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 21121024, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.17.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 3, |
| "offset": 21123072, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 22171648, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.17.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 22204416, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.17.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 3, |
| "offset": 22206464, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 23255040, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.17.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 23287808, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.17.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 3, |
| "offset": 23296000, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 27490304, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.17.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 27621376, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.17.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "data": { |
| "shard": 3, |
| "offset": 27623424, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 31817728, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.17.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 31948800, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.17.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 31950848, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.17.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 31952896, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.17.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 31954944, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.17.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 31956992, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.17.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 3, |
| "offset": 31959040, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 33007616, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.17.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 33040384, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.17.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 3, |
| "offset": 33046528, |
| "bytes": 3145728 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 36192256, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.18.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 36290560, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.18.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 3, |
| "offset": 36302848, |
| "bytes": 6291456 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 42594304, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.18.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 42790912, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.18.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 768, |
| "padded_rows": 2048, |
| "padded_columns": 768, |
| "data": { |
| "shard": 3, |
| "offset": 42795008, |
| "bytes": 1572864 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 44367872, |
| "bytes": 49152 |
| } |
| }, |
| "blocks.18.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 44417024, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.18.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 3, |
| "offset": 44419072, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 45467648, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.18.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 45500416, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.18.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 3, |
| "offset": 45502464, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 46551040, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.18.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 46583808, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.18.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 3, |
| "offset": 46592000, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 50786304, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.18.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 50917376, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.18.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "data": { |
| "shard": 3, |
| "offset": 50919424, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 55113728, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.18.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 55244800, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.18.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 55246848, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.18.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 55248896, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.18.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 55250944, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.18.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 55252992, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.18.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 3, |
| "offset": 55255040, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 56303616, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.18.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 56336384, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.18.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 3, |
| "offset": 56342528, |
| "bytes": 3145728 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 59488256, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.19.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 59586560, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.19.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 3, |
| "offset": 59598848, |
| "bytes": 6291456 |
| }, |
| "scales": { |
| "shard": 3, |
| "offset": 65890304, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.19.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 3, |
| "offset": 66086912, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.19.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 768, |
| "padded_rows": 2048, |
| "padded_columns": 768, |
| "data": { |
| "shard": 4, |
| "offset": 0, |
| "bytes": 1572864 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 1572864, |
| "bytes": 49152 |
| } |
| }, |
| "blocks.19.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 1622016, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.19.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 4, |
| "offset": 1624064, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 2672640, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.19.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 2705408, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.19.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 4, |
| "offset": 2707456, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 3756032, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.19.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 3788800, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.19.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 4, |
| "offset": 3796992, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 7991296, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.19.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 8122368, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.19.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "data": { |
| "shard": 4, |
| "offset": 8124416, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 12318720, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.19.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 12449792, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.19.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 12451840, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.19.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 12453888, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.19.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 12455936, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.19.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 12457984, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.19.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 4, |
| "offset": 12460032, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 13508608, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.19.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 13541376, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.19.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 4, |
| "offset": 13547520, |
| "bytes": 3145728 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 16693248, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.2.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 16791552, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.2.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 4, |
| "offset": 16803840, |
| "bytes": 6291456 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 23095296, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.2.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 23291904, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.2.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 768, |
| "padded_rows": 2048, |
| "padded_columns": 768, |
| "data": { |
| "shard": 4, |
| "offset": 23296000, |
| "bytes": 1572864 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 24868864, |
| "bytes": 49152 |
| } |
| }, |
| "blocks.2.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 24918016, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.2.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 4, |
| "offset": 24920064, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 25968640, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.2.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 26001408, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.2.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 4, |
| "offset": 26003456, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 27052032, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.2.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 27084800, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.2.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 4, |
| "offset": 27092992, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 31287296, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.2.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 31418368, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.2.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "data": { |
| "shard": 4, |
| "offset": 31420416, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 35614720, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.2.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 35745792, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.2.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 35747840, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.2.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 35749888, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.2.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 35751936, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.2.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 35753984, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.2.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 4, |
| "offset": 35756032, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 36804608, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.2.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 36837376, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.2.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 4, |
| "offset": 36843520, |
| "bytes": 3145728 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 39989248, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.20.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 40087552, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.20.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 4, |
| "offset": 40099840, |
| "bytes": 6291456 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 46391296, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.20.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 46587904, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.20.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 768, |
| "padded_rows": 2048, |
| "padded_columns": 768, |
| "data": { |
| "shard": 4, |
| "offset": 46592000, |
| "bytes": 1572864 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 48164864, |
| "bytes": 49152 |
| } |
| }, |
| "blocks.20.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 48214016, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.20.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 4, |
| "offset": 48216064, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 49264640, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.20.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 49297408, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.20.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 4, |
| "offset": 49299456, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 50348032, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.20.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 50380800, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.20.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 4, |
| "offset": 50388992, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 54583296, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.20.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 54714368, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.20.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "data": { |
| "shard": 4, |
| "offset": 54716416, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 58910720, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.20.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 59041792, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.20.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 59043840, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.20.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 59045888, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.20.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 59047936, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.20.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 59049984, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.20.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 4, |
| "offset": 59052032, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 60100608, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.20.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 60133376, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.20.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 4, |
| "offset": 60139520, |
| "bytes": 3145728 |
| }, |
| "scales": { |
| "shard": 4, |
| "offset": 63285248, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.21.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 4, |
| "offset": 63383552, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.21.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 5, |
| "offset": 0, |
| "bytes": 6291456 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 6291456, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.21.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 6488064, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.21.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 768, |
| "padded_rows": 2048, |
| "padded_columns": 768, |
| "data": { |
| "shard": 5, |
| "offset": 6492160, |
| "bytes": 1572864 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 8065024, |
| "bytes": 49152 |
| } |
| }, |
| "blocks.21.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 8114176, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.21.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 5, |
| "offset": 8116224, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 9164800, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.21.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 9197568, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.21.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 5, |
| "offset": 9199616, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 10248192, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.21.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 10280960, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.21.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 5, |
| "offset": 10289152, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 14483456, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.21.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 14614528, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.21.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "data": { |
| "shard": 5, |
| "offset": 14616576, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 18810880, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.21.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 18941952, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.21.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 18944000, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.21.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 18946048, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.21.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 18948096, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.21.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 18950144, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.21.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 5, |
| "offset": 18952192, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 20000768, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.21.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 20033536, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.21.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 5, |
| "offset": 20039680, |
| "bytes": 3145728 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 23185408, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.22.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 23283712, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.22.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 5, |
| "offset": 23296000, |
| "bytes": 6291456 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 29587456, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.22.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 29784064, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.22.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 768, |
| "padded_rows": 2048, |
| "padded_columns": 768, |
| "data": { |
| "shard": 5, |
| "offset": 29788160, |
| "bytes": 1572864 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 31361024, |
| "bytes": 49152 |
| } |
| }, |
| "blocks.22.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 31410176, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.22.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 5, |
| "offset": 31412224, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 32460800, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.22.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 32493568, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.22.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 5, |
| "offset": 32495616, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 33544192, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.22.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 33576960, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.22.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 5, |
| "offset": 33585152, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 37779456, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.22.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 37910528, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.22.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "data": { |
| "shard": 5, |
| "offset": 37912576, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 42106880, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.22.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 42237952, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.22.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 42240000, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.22.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 42242048, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.22.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 42244096, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.22.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 42246144, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.22.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 5, |
| "offset": 42248192, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 43296768, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.22.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 43329536, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.22.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 5, |
| "offset": 43335680, |
| "bytes": 3145728 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 46481408, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.23.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 46579712, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.23.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 5, |
| "offset": 46592000, |
| "bytes": 6291456 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 52883456, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.23.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 53080064, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.23.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 768, |
| "padded_rows": 2048, |
| "padded_columns": 768, |
| "data": { |
| "shard": 5, |
| "offset": 53084160, |
| "bytes": 1572864 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 54657024, |
| "bytes": 49152 |
| } |
| }, |
| "blocks.23.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 54706176, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.23.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 5, |
| "offset": 54708224, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 55756800, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.23.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 55789568, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.23.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 5, |
| "offset": 55791616, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 56840192, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.23.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 56872960, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.23.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 5, |
| "offset": 56881152, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 61075456, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.23.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 61206528, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.23.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "data": { |
| "shard": 5, |
| "offset": 61208576, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 65402880, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.23.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 65533952, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.23.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 65536000, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.23.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 65538048, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.23.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 65540096, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.23.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 65542144, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.23.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 5, |
| "offset": 65544192, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 5, |
| "offset": 66592768, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.23.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 5, |
| "offset": 66625536, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.23.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 6, |
| "offset": 0, |
| "bytes": 3145728 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 3145728, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.3.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 3244032, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.3.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 6, |
| "offset": 3256320, |
| "bytes": 6291456 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 9547776, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.3.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 9744384, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.3.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 768, |
| "padded_rows": 2048, |
| "padded_columns": 768, |
| "data": { |
| "shard": 6, |
| "offset": 9748480, |
| "bytes": 1572864 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 11321344, |
| "bytes": 49152 |
| } |
| }, |
| "blocks.3.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 11370496, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.3.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 6, |
| "offset": 11372544, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 12421120, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.3.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 12453888, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.3.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 6, |
| "offset": 12455936, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 13504512, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.3.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 13537280, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.3.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 6, |
| "offset": 13545472, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 17739776, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.3.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 17870848, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.3.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "data": { |
| "shard": 6, |
| "offset": 17872896, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 22067200, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.3.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 22198272, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.3.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 22200320, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.3.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 22202368, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.3.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 22204416, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.3.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 22206464, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.3.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 6, |
| "offset": 22208512, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 23257088, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.3.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 23289856, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.3.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 6, |
| "offset": 23296000, |
| "bytes": 3145728 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 26441728, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.4.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 26540032, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.4.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 6, |
| "offset": 26552320, |
| "bytes": 6291456 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 32843776, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.4.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 33040384, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.4.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 768, |
| "padded_rows": 2048, |
| "padded_columns": 768, |
| "data": { |
| "shard": 6, |
| "offset": 33044480, |
| "bytes": 1572864 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 34617344, |
| "bytes": 49152 |
| } |
| }, |
| "blocks.4.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 34666496, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.4.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 6, |
| "offset": 34668544, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 35717120, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.4.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 35749888, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.4.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 6, |
| "offset": 35751936, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 36800512, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.4.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 36833280, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.4.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 6, |
| "offset": 36841472, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 41035776, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.4.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 41166848, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.4.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "data": { |
| "shard": 6, |
| "offset": 41168896, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 45363200, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.4.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 45494272, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.4.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 45496320, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.4.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 45498368, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.4.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 45500416, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.4.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 45502464, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.4.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 6, |
| "offset": 45504512, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 46553088, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.4.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 46585856, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.4.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 6, |
| "offset": 46592000, |
| "bytes": 3145728 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 49737728, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.5.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 49836032, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.5.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 6, |
| "offset": 49848320, |
| "bytes": 6291456 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 56139776, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.5.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 56336384, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.5.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 768, |
| "padded_rows": 2048, |
| "padded_columns": 768, |
| "data": { |
| "shard": 6, |
| "offset": 56340480, |
| "bytes": 1572864 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 57913344, |
| "bytes": 49152 |
| } |
| }, |
| "blocks.5.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 57962496, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.5.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 6, |
| "offset": 57964544, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 59013120, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.5.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 59045888, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.5.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 6, |
| "offset": 59047936, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 60096512, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.5.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 60129280, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.5.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 6, |
| "offset": 60137472, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 6, |
| "offset": 64331776, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.5.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 6, |
| "offset": 64462848, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.5.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "data": { |
| "shard": 7, |
| "offset": 0, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 4194304, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.5.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 4325376, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.5.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 4327424, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.5.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 4329472, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.5.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 4331520, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.5.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 4333568, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.5.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 7, |
| "offset": 4335616, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 5384192, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.5.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 5416960, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.5.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 7, |
| "offset": 5423104, |
| "bytes": 3145728 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 8568832, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.6.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 8667136, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.6.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 7, |
| "offset": 8679424, |
| "bytes": 6291456 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 14970880, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.6.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 15167488, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.6.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 768, |
| "padded_rows": 2048, |
| "padded_columns": 768, |
| "data": { |
| "shard": 7, |
| "offset": 15171584, |
| "bytes": 1572864 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 16744448, |
| "bytes": 49152 |
| } |
| }, |
| "blocks.6.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 16793600, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.6.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 7, |
| "offset": 16795648, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 17844224, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.6.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 17876992, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.6.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 7, |
| "offset": 17879040, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 18927616, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.6.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 18960384, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.6.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 7, |
| "offset": 18968576, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 23162880, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.6.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 23293952, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.6.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "data": { |
| "shard": 7, |
| "offset": 23296000, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 27490304, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.6.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 27621376, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.6.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 27623424, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.6.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 27625472, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.6.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 27627520, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.6.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 27629568, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.6.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 7, |
| "offset": 27631616, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 28680192, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.6.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 28712960, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.6.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 7, |
| "offset": 28719104, |
| "bytes": 3145728 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 31864832, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.7.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 31963136, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.7.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 7, |
| "offset": 31975424, |
| "bytes": 6291456 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 38266880, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.7.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 38463488, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.7.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 768, |
| "padded_rows": 2048, |
| "padded_columns": 768, |
| "data": { |
| "shard": 7, |
| "offset": 38467584, |
| "bytes": 1572864 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 40040448, |
| "bytes": 49152 |
| } |
| }, |
| "blocks.7.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 40089600, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.7.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 7, |
| "offset": 40091648, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 41140224, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.7.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 41172992, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.7.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 7, |
| "offset": 41175040, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 42223616, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.7.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 42256384, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.7.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 7, |
| "offset": 42264576, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 46458880, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.7.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 46589952, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.7.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "data": { |
| "shard": 7, |
| "offset": 46592000, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 50786304, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.7.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 50917376, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.7.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 50919424, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.7.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 50921472, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.7.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 50923520, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.7.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 50925568, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.7.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 7, |
| "offset": 50927616, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 51976192, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.7.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 52008960, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.7.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 7, |
| "offset": 52015104, |
| "bytes": 3145728 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 55160832, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.8.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 55259136, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.8.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 7, |
| "offset": 55271424, |
| "bytes": 6291456 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 61562880, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.8.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 61759488, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.8.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 768, |
| "padded_rows": 2048, |
| "padded_columns": 768, |
| "data": { |
| "shard": 7, |
| "offset": 61763584, |
| "bytes": 1572864 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 63336448, |
| "bytes": 49152 |
| } |
| }, |
| "blocks.8.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 63385600, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.8.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 7, |
| "offset": 63387648, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 64436224, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.8.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 64468992, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.8.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 7, |
| "offset": 64471040, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 7, |
| "offset": 65519616, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.8.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 7, |
| "offset": 65552384, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.8.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 8, |
| "offset": 0, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 8, |
| "offset": 4194304, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.8.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 4325376, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.8.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "data": { |
| "shard": 8, |
| "offset": 4327424, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 8, |
| "offset": 8521728, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.8.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 8652800, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.8.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 8654848, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.8.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 8656896, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.8.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 8658944, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.8.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 8660992, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.8.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 8, |
| "offset": 8663040, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 8, |
| "offset": 9711616, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.8.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 9744384, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.8.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 8, |
| "offset": 9750528, |
| "bytes": 3145728 |
| }, |
| "scales": { |
| "shard": 8, |
| "offset": 12896256, |
| "bytes": 98304 |
| } |
| }, |
| "blocks.9.adaLN_modulation.1.bias": { |
| "shape": [ |
| 6144 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 12994560, |
| "bytes": 12288 |
| } |
| }, |
| "blocks.9.adaLN_modulation.1.weight": { |
| "shape": [ |
| 6144, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 6144, |
| "columns": 1024, |
| "padded_rows": 6144, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 8, |
| "offset": 13006848, |
| "bytes": 6291456 |
| }, |
| "scales": { |
| "shard": 8, |
| "offset": 19298304, |
| "bytes": 196608 |
| } |
| }, |
| "blocks.9.cross_attn.to_kv.bias": { |
| "shape": [ |
| 2048 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 19494912, |
| "bytes": 4096 |
| } |
| }, |
| "blocks.9.cross_attn.to_kv.weight": { |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 2048, |
| "columns": 768, |
| "padded_rows": 2048, |
| "padded_columns": 768, |
| "data": { |
| "shard": 8, |
| "offset": 19499008, |
| "bytes": 1572864 |
| }, |
| "scales": { |
| "shard": 8, |
| "offset": 21071872, |
| "bytes": 49152 |
| } |
| }, |
| "blocks.9.cross_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 21121024, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.9.cross_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 8, |
| "offset": 21123072, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 8, |
| "offset": 22171648, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.9.cross_attn.to_q.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 22204416, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.9.cross_attn.to_q.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 8, |
| "offset": 22206464, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 8, |
| "offset": 23255040, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.9.mlp.mlp.0.bias": { |
| "shape": [ |
| 4096 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 23287808, |
| "bytes": 8192 |
| } |
| }, |
| "blocks.9.mlp.mlp.0.weight": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 8, |
| "offset": 23296000, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 8, |
| "offset": 27490304, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.9.mlp.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 27621376, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.9.mlp.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 4096, |
| "padded_rows": 1024, |
| "padded_columns": 4096, |
| "data": { |
| "shard": 8, |
| "offset": 27623424, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 8, |
| "offset": 31817728, |
| "bytes": 131072 |
| } |
| }, |
| "blocks.9.norm2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 31948800, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.9.norm2.weight": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 31950848, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.9.self_attn.k_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 31952896, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.9.self_attn.q_rms_norm.gamma": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 31954944, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.9.self_attn.to_out.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 31956992, |
| "bytes": 2048 |
| } |
| }, |
| "blocks.9.self_attn.to_out.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 8, |
| "offset": 31959040, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 8, |
| "offset": 33007616, |
| "bytes": 32768 |
| } |
| }, |
| "blocks.9.self_attn.to_qkv.bias": { |
| "shape": [ |
| 3072 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 33040384, |
| "bytes": 6144 |
| } |
| }, |
| "blocks.9.self_attn.to_qkv.weight": { |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 3072, |
| "columns": 1024, |
| "padded_rows": 3072, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 8, |
| "offset": 33046528, |
| "bytes": 3145728 |
| }, |
| "scales": { |
| "shard": 8, |
| "offset": 36192256, |
| "bytes": 98304 |
| } |
| }, |
| "input_layer.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 36290560, |
| "bytes": 2048 |
| } |
| }, |
| "input_layer.weight": { |
| "shape": [ |
| 1024, |
| 8 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 8, |
| "padded_rows": 1024, |
| "padded_columns": 64, |
| "data": { |
| "shard": 8, |
| "offset": 36292608, |
| "bytes": 65536 |
| }, |
| "scales": { |
| "shard": 8, |
| "offset": 36358144, |
| "bytes": 2048 |
| } |
| }, |
| "out_layer.bias": { |
| "shape": [ |
| 8 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 36360192, |
| "bytes": 16 |
| } |
| }, |
| "out_layer.weight": { |
| "shape": [ |
| 8, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 8, |
| "columns": 1024, |
| "padded_rows": 16, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 8, |
| "offset": 36360448, |
| "bytes": 16384 |
| }, |
| "scales": { |
| "shard": 8, |
| "offset": 36376832, |
| "bytes": 512 |
| } |
| }, |
| "pos_emb": { |
| "shape": [ |
| 4096, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 4096, |
| "columns": 1024, |
| "padded_rows": 4096, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 8, |
| "offset": 36377344, |
| "bytes": 4194304 |
| }, |
| "scales": { |
| "shard": 8, |
| "offset": 40571648, |
| "bytes": 131072 |
| } |
| }, |
| "t_embedder.mlp.0.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 40702720, |
| "bytes": 2048 |
| } |
| }, |
| "t_embedder.mlp.0.weight": { |
| "shape": [ |
| 1024, |
| 256 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 256, |
| "padded_rows": 1024, |
| "padded_columns": 256, |
| "data": { |
| "shard": 8, |
| "offset": 40704768, |
| "bytes": 262144 |
| }, |
| "scales": { |
| "shard": 8, |
| "offset": 40966912, |
| "bytes": 8192 |
| } |
| }, |
| "t_embedder.mlp.2.bias": { |
| "shape": [ |
| 1024 |
| ], |
| "encoding": "raw", |
| "dtype": "f16", |
| "data": { |
| "shard": 8, |
| "offset": 40975104, |
| "bytes": 2048 |
| } |
| }, |
| "t_embedder.mlp.2.weight": { |
| "shape": [ |
| 1024, |
| 1024 |
| ], |
| "encoding": "symmetric-quantized", |
| "bits": 8, |
| "group_size": 64, |
| "output_tile": 16, |
| "rows": 1024, |
| "columns": 1024, |
| "padded_rows": 1024, |
| "padded_columns": 1024, |
| "data": { |
| "shard": 8, |
| "offset": 40977152, |
| "bytes": 1048576 |
| }, |
| "scales": { |
| "shard": 8, |
| "offset": 42025728, |
| "bytes": 32768 |
| } |
| } |
| } |
| } |
|
|
