diff --git "a/image-large/sparseStructureFlow/manifest.json" "b/image-large/sparseStructureFlow/manifest.json"
new file mode 100644--- /dev/null
+++ "b/image-large/sparseStructureFlow/manifest.json"
@@ -0,0 +1,10104 @@
+{
+  "schema": "trellis-webgpu-pack",
+  "version": 1,
+  "stage": "sparseStructureFlow",
+  "upstreamRevision": "25e0d31ffbebe4b5a97464dd851910efc3002d96",
+  "shards": [
+    {
+      "file": "stage-000.bin",
+      "bytes": 66705408,
+      "sha256": "f0784b06e2b1de485881fb7382f4fb5f12ad1afb3bbf5a57cdbba9701b0a84b2"
+    },
+    {
+      "file": "stage-001.bin",
+      "bytes": 64538624,
+      "sha256": "44b727fea9979fd8f0d93a9f8edb3e8c37d380bcabd5fb4bd6cdc05903fa84e7"
+    },
+    {
+      "file": "stage-002.bin",
+      "bytes": 65628160,
+      "sha256": "2569e48e2700140e674038592de2f95440f1ed4f7897305300fe1cfd5f7b7e19"
+    },
+    {
+      "file": "stage-003.bin",
+      "bytes": 65617920,
+      "sha256": "3913321cb4f6dd6fc47619fc5c16a1369d28a5342fde66afa6f55d570468966e"
+    },
+    {
+      "file": "stage-004.bin",
+      "bytes": 62349312,
+      "sha256": "187edbe3a8d1258a2b2f15307a0c19a60608c9111aa98e13764935e7ba645f4f"
+    },
+    {
+      "file": "stage-005.bin",
+      "bytes": 66693120,
+      "sha256": "a1b1300cd56ac0379cbde82c8e5cf7ee17b81610db5a9bc36a6a8d0c9a36f7a7"
+    },
+    {
+      "file": "stage-006.bin",
+      "bytes": 64538624,
+      "sha256": "521267c28bfa739213e478c017db00eee2b42572c8b3f9c1e55808b65d74dd45"
+    },
+    {
+      "file": "stage-007.bin",
+      "bytes": 65628160,
+      "sha256": "0a1e8e93c5f80670cc0bca860d26a4d92e82c307707a41ef1bc0ced957b5e56f"
+    },
+    {
+      "file": "stage-008.bin",
+      "bytes": 65617920,
+      "sha256": "810a6c3f441ec717975ad7173e269178020f91ca307e703cb0d01f454a1c886e"
+    },
+    {
+      "file": "stage-009.bin",
+      "bytes": 62349312,
+      "sha256": "215a57b9568b49fd7b049780ddd563bb109fb72dc8754c7798420e03bb744657"
+    },
+    {
+      "file": "stage-010.bin",
+      "bytes": 66693120,
+      "sha256": "407fa67941be2024845c160fe3ccbe8c684cfe906e53b2c62704b97012a80898"
+    },
+    {
+      "file": "stage-011.bin",
+      "bytes": 64538624,
+      "sha256": "3fb0f00dbe8b4a45adcab7b36f298d1f975fef4f9249ac35bc4c8f1b37bd800f"
+    },
+    {
+      "file": "stage-012.bin",
+      "bytes": 65628160,
+      "sha256": "ea04edc07730f3dbd4f0df87a07d0027a57d7ed7513475f96878c90fc34d8f75"
+    },
+    {
+      "file": "stage-013.bin",
+      "bytes": 26177792,
+      "sha256": "cb7118a3f8a9dbbd2b7a4a4f83ef707ac2dbea1ce3f6dedb51457d4e857d969a"
+    }
+  ],
+  "tensors": {
+    "blocks.0.adaLN_modulation.1.bias": {
+      "shape": [
+        6144
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 0,
+        "offset": 0,
+        "bytes": 12288
+      }
+    },
+    "blocks.0.adaLN_modulation.1.weight": {
+      "shape": [
+        6144,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 6144,
+      "columns": 1024,
+      "padded_rows": 6144,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 0,
+        "offset": 12288,
+        "bytes": 6291456
+      },
+      "base_scales": {
+        "shard": 0,
+        "offset": 6303744,
+        "bytes": 196608
+      },
+      "residual_data": {
+        "shard": 0,
+        "offset": 6500352,
+        "bytes": 3145728
+      },
+      "residual_scales": {
+        "shard": 0,
+        "offset": 9646080,
+        "bytes": 196608
+      }
+    },
+    "blocks.0.cross_attn.to_kv.bias": {
+      "shape": [
+        2048
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 0,
+        "offset": 9842688,
+        "bytes": 4096
+      }
+    },
+    "blocks.0.cross_attn.to_kv.weight": {
+      "shape": [
+        2048,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 2048,
+      "columns": 1024,
+      "padded_rows": 2048,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 0,
+        "offset": 9846784,
+        "bytes": 2097152
+      },
+      "base_scales": {
+        "shard": 0,
+        "offset": 11943936,
+        "bytes": 65536
+      },
+      "residual_data": {
+        "shard": 0,
+        "offset": 12009472,
+        "bytes": 1048576
+      },
+      "residual_scales": {
+        "shard": 0,
+        "offset": 13058048,
+        "bytes": 65536
+      }
+    },
+    "blocks.0.cross_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 0,
+        "offset": 13123584,
+        "bytes": 2048
+      }
+    },
+    "blocks.0.cross_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 0,
+        "offset": 13125632,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 0,
+        "offset": 14174208,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 0,
+        "offset": 14206976,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 0,
+        "offset": 14731264,
+        "bytes": 32768
+      }
+    },
+    "blocks.0.cross_attn.to_q.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 0,
+        "offset": 14764032,
+        "bytes": 2048
+      }
+    },
+    "blocks.0.cross_attn.to_q.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 0,
+        "offset": 14766080,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 0,
+        "offset": 15814656,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 0,
+        "offset": 15847424,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 0,
+        "offset": 16371712,
+        "bytes": 32768
+      }
+    },
+    "blocks.0.mlp.mlp.0.bias": {
+      "shape": [
+        4096
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 0,
+        "offset": 16404480,
+        "bytes": 8192
+      }
+    },
+    "blocks.0.mlp.mlp.0.weight": {
+      "shape": [
+        4096,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 4096,
+      "columns": 1024,
+      "padded_rows": 4096,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 0,
+        "offset": 16412672,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 0,
+        "offset": 20606976,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 0,
+        "offset": 20738048,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 0,
+        "offset": 22835200,
+        "bytes": 131072
+      }
+    },
+    "blocks.0.mlp.mlp.2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 0,
+        "offset": 22966272,
+        "bytes": 2048
+      }
+    },
+    "blocks.0.mlp.mlp.2.weight": {
+      "shape": [
+        1024,
+        4096
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 4096,
+      "padded_rows": 1024,
+      "padded_columns": 4096,
+      "base_data": {
+        "shard": 0,
+        "offset": 22968320,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 0,
+        "offset": 27162624,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 0,
+        "offset": 27293696,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 0,
+        "offset": 29390848,
+        "bytes": 131072
+      }
+    },
+    "blocks.0.norm2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 0,
+        "offset": 29521920,
+        "bytes": 2048
+      }
+    },
+    "blocks.0.norm2.weight": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 0,
+        "offset": 29523968,
+        "bytes": 2048
+      }
+    },
+    "blocks.0.self_attn.k_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 0,
+        "offset": 29526016,
+        "bytes": 2048
+      }
+    },
+    "blocks.0.self_attn.q_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 0,
+        "offset": 29528064,
+        "bytes": 2048
+      }
+    },
+    "blocks.0.self_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 0,
+        "offset": 29530112,
+        "bytes": 2048
+      }
+    },
+    "blocks.0.self_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 0,
+        "offset": 29532160,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 0,
+        "offset": 30580736,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 0,
+        "offset": 30613504,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 0,
+        "offset": 31137792,
+        "bytes": 32768
+      }
+    },
+    "blocks.0.self_attn.to_qkv.bias": {
+      "shape": [
+        3072
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 0,
+        "offset": 31170560,
+        "bytes": 6144
+      }
+    },
+    "blocks.0.self_attn.to_qkv.weight": {
+      "shape": [
+        3072,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 3072,
+      "columns": 1024,
+      "padded_rows": 3072,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 0,
+        "offset": 31176704,
+        "bytes": 3145728
+      },
+      "base_scales": {
+        "shard": 0,
+        "offset": 34322432,
+        "bytes": 98304
+      },
+      "residual_data": {
+        "shard": 0,
+        "offset": 34420736,
+        "bytes": 1572864
+      },
+      "residual_scales": {
+        "shard": 0,
+        "offset": 35993600,
+        "bytes": 98304
+      }
+    },
+    "blocks.1.adaLN_modulation.1.bias": {
+      "shape": [
+        6144
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 0,
+        "offset": 36091904,
+        "bytes": 12288
+      }
+    },
+    "blocks.1.adaLN_modulation.1.weight": {
+      "shape": [
+        6144,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 6144,
+      "columns": 1024,
+      "padded_rows": 6144,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 0,
+        "offset": 36104192,
+        "bytes": 6291456
+      },
+      "base_scales": {
+        "shard": 0,
+        "offset": 42395648,
+        "bytes": 196608
+      },
+      "residual_data": {
+        "shard": 0,
+        "offset": 42592256,
+        "bytes": 3145728
+      },
+      "residual_scales": {
+        "shard": 0,
+        "offset": 45737984,
+        "bytes": 196608
+      }
+    },
+    "blocks.1.cross_attn.to_kv.bias": {
+      "shape": [
+        2048
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 0,
+        "offset": 45934592,
+        "bytes": 4096
+      }
+    },
+    "blocks.1.cross_attn.to_kv.weight": {
+      "shape": [
+        2048,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 2048,
+      "columns": 1024,
+      "padded_rows": 2048,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 0,
+        "offset": 45938688,
+        "bytes": 2097152
+      },
+      "base_scales": {
+        "shard": 0,
+        "offset": 48035840,
+        "bytes": 65536
+      },
+      "residual_data": {
+        "shard": 0,
+        "offset": 48101376,
+        "bytes": 1048576
+      },
+      "residual_scales": {
+        "shard": 0,
+        "offset": 49149952,
+        "bytes": 65536
+      }
+    },
+    "blocks.1.cross_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 0,
+        "offset": 49215488,
+        "bytes": 2048
+      }
+    },
+    "blocks.1.cross_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 0,
+        "offset": 49217536,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 0,
+        "offset": 50266112,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 0,
+        "offset": 50298880,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 0,
+        "offset": 50823168,
+        "bytes": 32768
+      }
+    },
+    "blocks.1.cross_attn.to_q.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 0,
+        "offset": 50855936,
+        "bytes": 2048
+      }
+    },
+    "blocks.1.cross_attn.to_q.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 0,
+        "offset": 50857984,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 0,
+        "offset": 51906560,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 0,
+        "offset": 51939328,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 0,
+        "offset": 52463616,
+        "bytes": 32768
+      }
+    },
+    "blocks.1.mlp.mlp.0.bias": {
+      "shape": [
+        4096
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 0,
+        "offset": 52496384,
+        "bytes": 8192
+      }
+    },
+    "blocks.1.mlp.mlp.0.weight": {
+      "shape": [
+        4096,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 4096,
+      "columns": 1024,
+      "padded_rows": 4096,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 0,
+        "offset": 52504576,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 0,
+        "offset": 56698880,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 0,
+        "offset": 56829952,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 0,
+        "offset": 58927104,
+        "bytes": 131072
+      }
+    },
+    "blocks.1.mlp.mlp.2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 0,
+        "offset": 59058176,
+        "bytes": 2048
+      }
+    },
+    "blocks.1.mlp.mlp.2.weight": {
+      "shape": [
+        1024,
+        4096
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 4096,
+      "padded_rows": 1024,
+      "padded_columns": 4096,
+      "base_data": {
+        "shard": 0,
+        "offset": 59060224,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 0,
+        "offset": 63254528,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 0,
+        "offset": 63385600,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 0,
+        "offset": 65482752,
+        "bytes": 131072
+      }
+    },
+    "blocks.1.norm2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 0,
+        "offset": 65613824,
+        "bytes": 2048
+      }
+    },
+    "blocks.1.norm2.weight": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 0,
+        "offset": 65615872,
+        "bytes": 2048
+      }
+    },
+    "blocks.1.self_attn.k_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 0,
+        "offset": 65617920,
+        "bytes": 2048
+      }
+    },
+    "blocks.1.self_attn.q_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 0,
+        "offset": 65619968,
+        "bytes": 2048
+      }
+    },
+    "blocks.1.self_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 0,
+        "offset": 65622016,
+        "bytes": 2048
+      }
+    },
+    "blocks.1.self_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 0,
+        "offset": 65624064,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 0,
+        "offset": 66672640,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 1,
+        "offset": 0,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 1,
+        "offset": 524288,
+        "bytes": 32768
+      }
+    },
+    "blocks.1.self_attn.to_qkv.bias": {
+      "shape": [
+        3072
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 1,
+        "offset": 557056,
+        "bytes": 6144
+      }
+    },
+    "blocks.1.self_attn.to_qkv.weight": {
+      "shape": [
+        3072,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 3072,
+      "columns": 1024,
+      "padded_rows": 3072,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 1,
+        "offset": 563200,
+        "bytes": 3145728
+      },
+      "base_scales": {
+        "shard": 1,
+        "offset": 3708928,
+        "bytes": 98304
+      },
+      "residual_data": {
+        "shard": 1,
+        "offset": 3807232,
+        "bytes": 1572864
+      },
+      "residual_scales": {
+        "shard": 1,
+        "offset": 5380096,
+        "bytes": 98304
+      }
+    },
+    "blocks.10.adaLN_modulation.1.bias": {
+      "shape": [
+        6144
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 1,
+        "offset": 5478400,
+        "bytes": 12288
+      }
+    },
+    "blocks.10.adaLN_modulation.1.weight": {
+      "shape": [
+        6144,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 6144,
+      "columns": 1024,
+      "padded_rows": 6144,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 1,
+        "offset": 5490688,
+        "bytes": 6291456
+      },
+      "base_scales": {
+        "shard": 1,
+        "offset": 11782144,
+        "bytes": 196608
+      },
+      "residual_data": {
+        "shard": 1,
+        "offset": 11978752,
+        "bytes": 3145728
+      },
+      "residual_scales": {
+        "shard": 1,
+        "offset": 15124480,
+        "bytes": 196608
+      }
+    },
+    "blocks.10.cross_attn.to_kv.bias": {
+      "shape": [
+        2048
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 1,
+        "offset": 15321088,
+        "bytes": 4096
+      }
+    },
+    "blocks.10.cross_attn.to_kv.weight": {
+      "shape": [
+        2048,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 2048,
+      "columns": 1024,
+      "padded_rows": 2048,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 1,
+        "offset": 15325184,
+        "bytes": 2097152
+      },
+      "base_scales": {
+        "shard": 1,
+        "offset": 17422336,
+        "bytes": 65536
+      },
+      "residual_data": {
+        "shard": 1,
+        "offset": 17487872,
+        "bytes": 1048576
+      },
+      "residual_scales": {
+        "shard": 1,
+        "offset": 18536448,
+        "bytes": 65536
+      }
+    },
+    "blocks.10.cross_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 1,
+        "offset": 18601984,
+        "bytes": 2048
+      }
+    },
+    "blocks.10.cross_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 1,
+        "offset": 18604032,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 1,
+        "offset": 19652608,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 1,
+        "offset": 19685376,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 1,
+        "offset": 20209664,
+        "bytes": 32768
+      }
+    },
+    "blocks.10.cross_attn.to_q.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 1,
+        "offset": 20242432,
+        "bytes": 2048
+      }
+    },
+    "blocks.10.cross_attn.to_q.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 1,
+        "offset": 20244480,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 1,
+        "offset": 21293056,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 1,
+        "offset": 21325824,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 1,
+        "offset": 21850112,
+        "bytes": 32768
+      }
+    },
+    "blocks.10.mlp.mlp.0.bias": {
+      "shape": [
+        4096
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 1,
+        "offset": 21882880,
+        "bytes": 8192
+      }
+    },
+    "blocks.10.mlp.mlp.0.weight": {
+      "shape": [
+        4096,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 4096,
+      "columns": 1024,
+      "padded_rows": 4096,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 1,
+        "offset": 21891072,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 1,
+        "offset": 26085376,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 1,
+        "offset": 26216448,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 1,
+        "offset": 28313600,
+        "bytes": 131072
+      }
+    },
+    "blocks.10.mlp.mlp.2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 1,
+        "offset": 28444672,
+        "bytes": 2048
+      }
+    },
+    "blocks.10.mlp.mlp.2.weight": {
+      "shape": [
+        1024,
+        4096
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 4096,
+      "padded_rows": 1024,
+      "padded_columns": 4096,
+      "base_data": {
+        "shard": 1,
+        "offset": 28446720,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 1,
+        "offset": 32641024,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 1,
+        "offset": 32772096,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 1,
+        "offset": 34869248,
+        "bytes": 131072
+      }
+    },
+    "blocks.10.norm2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 1,
+        "offset": 35000320,
+        "bytes": 2048
+      }
+    },
+    "blocks.10.norm2.weight": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 1,
+        "offset": 35002368,
+        "bytes": 2048
+      }
+    },
+    "blocks.10.self_attn.k_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 1,
+        "offset": 35004416,
+        "bytes": 2048
+      }
+    },
+    "blocks.10.self_attn.q_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 1,
+        "offset": 35006464,
+        "bytes": 2048
+      }
+    },
+    "blocks.10.self_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 1,
+        "offset": 35008512,
+        "bytes": 2048
+      }
+    },
+    "blocks.10.self_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 1,
+        "offset": 35010560,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 1,
+        "offset": 36059136,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 1,
+        "offset": 36091904,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 1,
+        "offset": 36616192,
+        "bytes": 32768
+      }
+    },
+    "blocks.10.self_attn.to_qkv.bias": {
+      "shape": [
+        3072
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 1,
+        "offset": 36648960,
+        "bytes": 6144
+      }
+    },
+    "blocks.10.self_attn.to_qkv.weight": {
+      "shape": [
+        3072,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 3072,
+      "columns": 1024,
+      "padded_rows": 3072,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 1,
+        "offset": 36655104,
+        "bytes": 3145728
+      },
+      "base_scales": {
+        "shard": 1,
+        "offset": 39800832,
+        "bytes": 98304
+      },
+      "residual_data": {
+        "shard": 1,
+        "offset": 39899136,
+        "bytes": 1572864
+      },
+      "residual_scales": {
+        "shard": 1,
+        "offset": 41472000,
+        "bytes": 98304
+      }
+    },
+    "blocks.11.adaLN_modulation.1.bias": {
+      "shape": [
+        6144
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 1,
+        "offset": 41570304,
+        "bytes": 12288
+      }
+    },
+    "blocks.11.adaLN_modulation.1.weight": {
+      "shape": [
+        6144,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 6144,
+      "columns": 1024,
+      "padded_rows": 6144,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 1,
+        "offset": 41582592,
+        "bytes": 6291456
+      },
+      "base_scales": {
+        "shard": 1,
+        "offset": 47874048,
+        "bytes": 196608
+      },
+      "residual_data": {
+        "shard": 1,
+        "offset": 48070656,
+        "bytes": 3145728
+      },
+      "residual_scales": {
+        "shard": 1,
+        "offset": 51216384,
+        "bytes": 196608
+      }
+    },
+    "blocks.11.cross_attn.to_kv.bias": {
+      "shape": [
+        2048
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 1,
+        "offset": 51412992,
+        "bytes": 4096
+      }
+    },
+    "blocks.11.cross_attn.to_kv.weight": {
+      "shape": [
+        2048,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 2048,
+      "columns": 1024,
+      "padded_rows": 2048,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 1,
+        "offset": 51417088,
+        "bytes": 2097152
+      },
+      "base_scales": {
+        "shard": 1,
+        "offset": 53514240,
+        "bytes": 65536
+      },
+      "residual_data": {
+        "shard": 1,
+        "offset": 53579776,
+        "bytes": 1048576
+      },
+      "residual_scales": {
+        "shard": 1,
+        "offset": 54628352,
+        "bytes": 65536
+      }
+    },
+    "blocks.11.cross_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 1,
+        "offset": 54693888,
+        "bytes": 2048
+      }
+    },
+    "blocks.11.cross_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 1,
+        "offset": 54695936,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 1,
+        "offset": 55744512,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 1,
+        "offset": 55777280,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 1,
+        "offset": 56301568,
+        "bytes": 32768
+      }
+    },
+    "blocks.11.cross_attn.to_q.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 1,
+        "offset": 56334336,
+        "bytes": 2048
+      }
+    },
+    "blocks.11.cross_attn.to_q.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 1,
+        "offset": 56336384,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 1,
+        "offset": 57384960,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 1,
+        "offset": 57417728,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 1,
+        "offset": 57942016,
+        "bytes": 32768
+      }
+    },
+    "blocks.11.mlp.mlp.0.bias": {
+      "shape": [
+        4096
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 1,
+        "offset": 57974784,
+        "bytes": 8192
+      }
+    },
+    "blocks.11.mlp.mlp.0.weight": {
+      "shape": [
+        4096,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 4096,
+      "columns": 1024,
+      "padded_rows": 4096,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 1,
+        "offset": 57982976,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 1,
+        "offset": 62177280,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 1,
+        "offset": 62308352,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 1,
+        "offset": 64405504,
+        "bytes": 131072
+      }
+    },
+    "blocks.11.mlp.mlp.2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 1,
+        "offset": 64536576,
+        "bytes": 2048
+      }
+    },
+    "blocks.11.mlp.mlp.2.weight": {
+      "shape": [
+        1024,
+        4096
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 4096,
+      "padded_rows": 1024,
+      "padded_columns": 4096,
+      "base_data": {
+        "shard": 2,
+        "offset": 0,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 2,
+        "offset": 4194304,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 2,
+        "offset": 4325376,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 2,
+        "offset": 6422528,
+        "bytes": 131072
+      }
+    },
+    "blocks.11.norm2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 2,
+        "offset": 6553600,
+        "bytes": 2048
+      }
+    },
+    "blocks.11.norm2.weight": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 2,
+        "offset": 6555648,
+        "bytes": 2048
+      }
+    },
+    "blocks.11.self_attn.k_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 2,
+        "offset": 6557696,
+        "bytes": 2048
+      }
+    },
+    "blocks.11.self_attn.q_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 2,
+        "offset": 6559744,
+        "bytes": 2048
+      }
+    },
+    "blocks.11.self_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 2,
+        "offset": 6561792,
+        "bytes": 2048
+      }
+    },
+    "blocks.11.self_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 2,
+        "offset": 6563840,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 2,
+        "offset": 7612416,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 2,
+        "offset": 7645184,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 2,
+        "offset": 8169472,
+        "bytes": 32768
+      }
+    },
+    "blocks.11.self_attn.to_qkv.bias": {
+      "shape": [
+        3072
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 2,
+        "offset": 8202240,
+        "bytes": 6144
+      }
+    },
+    "blocks.11.self_attn.to_qkv.weight": {
+      "shape": [
+        3072,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 3072,
+      "columns": 1024,
+      "padded_rows": 3072,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 2,
+        "offset": 8208384,
+        "bytes": 3145728
+      },
+      "base_scales": {
+        "shard": 2,
+        "offset": 11354112,
+        "bytes": 98304
+      },
+      "residual_data": {
+        "shard": 2,
+        "offset": 11452416,
+        "bytes": 1572864
+      },
+      "residual_scales": {
+        "shard": 2,
+        "offset": 13025280,
+        "bytes": 98304
+      }
+    },
+    "blocks.12.adaLN_modulation.1.bias": {
+      "shape": [
+        6144
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 2,
+        "offset": 13123584,
+        "bytes": 12288
+      }
+    },
+    "blocks.12.adaLN_modulation.1.weight": {
+      "shape": [
+        6144,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 6144,
+      "columns": 1024,
+      "padded_rows": 6144,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 2,
+        "offset": 13135872,
+        "bytes": 6291456
+      },
+      "base_scales": {
+        "shard": 2,
+        "offset": 19427328,
+        "bytes": 196608
+      },
+      "residual_data": {
+        "shard": 2,
+        "offset": 19623936,
+        "bytes": 3145728
+      },
+      "residual_scales": {
+        "shard": 2,
+        "offset": 22769664,
+        "bytes": 196608
+      }
+    },
+    "blocks.12.cross_attn.to_kv.bias": {
+      "shape": [
+        2048
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 2,
+        "offset": 22966272,
+        "bytes": 4096
+      }
+    },
+    "blocks.12.cross_attn.to_kv.weight": {
+      "shape": [
+        2048,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 2048,
+      "columns": 1024,
+      "padded_rows": 2048,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 2,
+        "offset": 22970368,
+        "bytes": 2097152
+      },
+      "base_scales": {
+        "shard": 2,
+        "offset": 25067520,
+        "bytes": 65536
+      },
+      "residual_data": {
+        "shard": 2,
+        "offset": 25133056,
+        "bytes": 1048576
+      },
+      "residual_scales": {
+        "shard": 2,
+        "offset": 26181632,
+        "bytes": 65536
+      }
+    },
+    "blocks.12.cross_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 2,
+        "offset": 26247168,
+        "bytes": 2048
+      }
+    },
+    "blocks.12.cross_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 2,
+        "offset": 26249216,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 2,
+        "offset": 27297792,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 2,
+        "offset": 27330560,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 2,
+        "offset": 27854848,
+        "bytes": 32768
+      }
+    },
+    "blocks.12.cross_attn.to_q.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 2,
+        "offset": 27887616,
+        "bytes": 2048
+      }
+    },
+    "blocks.12.cross_attn.to_q.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 2,
+        "offset": 27889664,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 2,
+        "offset": 28938240,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 2,
+        "offset": 28971008,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 2,
+        "offset": 29495296,
+        "bytes": 32768
+      }
+    },
+    "blocks.12.mlp.mlp.0.bias": {
+      "shape": [
+        4096
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 2,
+        "offset": 29528064,
+        "bytes": 8192
+      }
+    },
+    "blocks.12.mlp.mlp.0.weight": {
+      "shape": [
+        4096,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 4096,
+      "columns": 1024,
+      "padded_rows": 4096,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 2,
+        "offset": 29536256,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 2,
+        "offset": 33730560,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 2,
+        "offset": 33861632,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 2,
+        "offset": 35958784,
+        "bytes": 131072
+      }
+    },
+    "blocks.12.mlp.mlp.2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 2,
+        "offset": 36089856,
+        "bytes": 2048
+      }
+    },
+    "blocks.12.mlp.mlp.2.weight": {
+      "shape": [
+        1024,
+        4096
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 4096,
+      "padded_rows": 1024,
+      "padded_columns": 4096,
+      "base_data": {
+        "shard": 2,
+        "offset": 36091904,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 2,
+        "offset": 40286208,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 2,
+        "offset": 40417280,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 2,
+        "offset": 42514432,
+        "bytes": 131072
+      }
+    },
+    "blocks.12.norm2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 2,
+        "offset": 42645504,
+        "bytes": 2048
+      }
+    },
+    "blocks.12.norm2.weight": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 2,
+        "offset": 42647552,
+        "bytes": 2048
+      }
+    },
+    "blocks.12.self_attn.k_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 2,
+        "offset": 42649600,
+        "bytes": 2048
+      }
+    },
+    "blocks.12.self_attn.q_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 2,
+        "offset": 42651648,
+        "bytes": 2048
+      }
+    },
+    "blocks.12.self_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 2,
+        "offset": 42653696,
+        "bytes": 2048
+      }
+    },
+    "blocks.12.self_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 2,
+        "offset": 42655744,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 2,
+        "offset": 43704320,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 2,
+        "offset": 43737088,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 2,
+        "offset": 44261376,
+        "bytes": 32768
+      }
+    },
+    "blocks.12.self_attn.to_qkv.bias": {
+      "shape": [
+        3072
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 2,
+        "offset": 44294144,
+        "bytes": 6144
+      }
+    },
+    "blocks.12.self_attn.to_qkv.weight": {
+      "shape": [
+        3072,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 3072,
+      "columns": 1024,
+      "padded_rows": 3072,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 2,
+        "offset": 44300288,
+        "bytes": 3145728
+      },
+      "base_scales": {
+        "shard": 2,
+        "offset": 47446016,
+        "bytes": 98304
+      },
+      "residual_data": {
+        "shard": 2,
+        "offset": 47544320,
+        "bytes": 1572864
+      },
+      "residual_scales": {
+        "shard": 2,
+        "offset": 49117184,
+        "bytes": 98304
+      }
+    },
+    "blocks.13.adaLN_modulation.1.bias": {
+      "shape": [
+        6144
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 2,
+        "offset": 49215488,
+        "bytes": 12288
+      }
+    },
+    "blocks.13.adaLN_modulation.1.weight": {
+      "shape": [
+        6144,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 6144,
+      "columns": 1024,
+      "padded_rows": 6144,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 2,
+        "offset": 49227776,
+        "bytes": 6291456
+      },
+      "base_scales": {
+        "shard": 2,
+        "offset": 55519232,
+        "bytes": 196608
+      },
+      "residual_data": {
+        "shard": 2,
+        "offset": 55715840,
+        "bytes": 3145728
+      },
+      "residual_scales": {
+        "shard": 2,
+        "offset": 58861568,
+        "bytes": 196608
+      }
+    },
+    "blocks.13.cross_attn.to_kv.bias": {
+      "shape": [
+        2048
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 2,
+        "offset": 59058176,
+        "bytes": 4096
+      }
+    },
+    "blocks.13.cross_attn.to_kv.weight": {
+      "shape": [
+        2048,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 2048,
+      "columns": 1024,
+      "padded_rows": 2048,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 2,
+        "offset": 59062272,
+        "bytes": 2097152
+      },
+      "base_scales": {
+        "shard": 2,
+        "offset": 61159424,
+        "bytes": 65536
+      },
+      "residual_data": {
+        "shard": 2,
+        "offset": 61224960,
+        "bytes": 1048576
+      },
+      "residual_scales": {
+        "shard": 2,
+        "offset": 62273536,
+        "bytes": 65536
+      }
+    },
+    "blocks.13.cross_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 2,
+        "offset": 62339072,
+        "bytes": 2048
+      }
+    },
+    "blocks.13.cross_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 2,
+        "offset": 62341120,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 2,
+        "offset": 63389696,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 2,
+        "offset": 63422464,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 2,
+        "offset": 63946752,
+        "bytes": 32768
+      }
+    },
+    "blocks.13.cross_attn.to_q.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 2,
+        "offset": 63979520,
+        "bytes": 2048
+      }
+    },
+    "blocks.13.cross_attn.to_q.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 2,
+        "offset": 63981568,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 2,
+        "offset": 65030144,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 2,
+        "offset": 65062912,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 2,
+        "offset": 65587200,
+        "bytes": 32768
+      }
+    },
+    "blocks.13.mlp.mlp.0.bias": {
+      "shape": [
+        4096
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 2,
+        "offset": 65619968,
+        "bytes": 8192
+      }
+    },
+    "blocks.13.mlp.mlp.0.weight": {
+      "shape": [
+        4096,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 4096,
+      "columns": 1024,
+      "padded_rows": 4096,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 3,
+        "offset": 0,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 3,
+        "offset": 4194304,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 3,
+        "offset": 4325376,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 3,
+        "offset": 6422528,
+        "bytes": 131072
+      }
+    },
+    "blocks.13.mlp.mlp.2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 3,
+        "offset": 6553600,
+        "bytes": 2048
+      }
+    },
+    "blocks.13.mlp.mlp.2.weight": {
+      "shape": [
+        1024,
+        4096
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 4096,
+      "padded_rows": 1024,
+      "padded_columns": 4096,
+      "base_data": {
+        "shard": 3,
+        "offset": 6555648,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 3,
+        "offset": 10749952,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 3,
+        "offset": 10881024,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 3,
+        "offset": 12978176,
+        "bytes": 131072
+      }
+    },
+    "blocks.13.norm2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 3,
+        "offset": 13109248,
+        "bytes": 2048
+      }
+    },
+    "blocks.13.norm2.weight": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 3,
+        "offset": 13111296,
+        "bytes": 2048
+      }
+    },
+    "blocks.13.self_attn.k_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 3,
+        "offset": 13113344,
+        "bytes": 2048
+      }
+    },
+    "blocks.13.self_attn.q_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 3,
+        "offset": 13115392,
+        "bytes": 2048
+      }
+    },
+    "blocks.13.self_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 3,
+        "offset": 13117440,
+        "bytes": 2048
+      }
+    },
+    "blocks.13.self_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 3,
+        "offset": 13119488,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 3,
+        "offset": 14168064,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 3,
+        "offset": 14200832,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 3,
+        "offset": 14725120,
+        "bytes": 32768
+      }
+    },
+    "blocks.13.self_attn.to_qkv.bias": {
+      "shape": [
+        3072
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 3,
+        "offset": 14757888,
+        "bytes": 6144
+      }
+    },
+    "blocks.13.self_attn.to_qkv.weight": {
+      "shape": [
+        3072,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 3072,
+      "columns": 1024,
+      "padded_rows": 3072,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 3,
+        "offset": 14764032,
+        "bytes": 3145728
+      },
+      "base_scales": {
+        "shard": 3,
+        "offset": 17909760,
+        "bytes": 98304
+      },
+      "residual_data": {
+        "shard": 3,
+        "offset": 18008064,
+        "bytes": 1572864
+      },
+      "residual_scales": {
+        "shard": 3,
+        "offset": 19580928,
+        "bytes": 98304
+      }
+    },
+    "blocks.14.adaLN_modulation.1.bias": {
+      "shape": [
+        6144
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 3,
+        "offset": 19679232,
+        "bytes": 12288
+      }
+    },
+    "blocks.14.adaLN_modulation.1.weight": {
+      "shape": [
+        6144,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 6144,
+      "columns": 1024,
+      "padded_rows": 6144,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 3,
+        "offset": 19691520,
+        "bytes": 6291456
+      },
+      "base_scales": {
+        "shard": 3,
+        "offset": 25982976,
+        "bytes": 196608
+      },
+      "residual_data": {
+        "shard": 3,
+        "offset": 26179584,
+        "bytes": 3145728
+      },
+      "residual_scales": {
+        "shard": 3,
+        "offset": 29325312,
+        "bytes": 196608
+      }
+    },
+    "blocks.14.cross_attn.to_kv.bias": {
+      "shape": [
+        2048
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 3,
+        "offset": 29521920,
+        "bytes": 4096
+      }
+    },
+    "blocks.14.cross_attn.to_kv.weight": {
+      "shape": [
+        2048,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 2048,
+      "columns": 1024,
+      "padded_rows": 2048,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 3,
+        "offset": 29526016,
+        "bytes": 2097152
+      },
+      "base_scales": {
+        "shard": 3,
+        "offset": 31623168,
+        "bytes": 65536
+      },
+      "residual_data": {
+        "shard": 3,
+        "offset": 31688704,
+        "bytes": 1048576
+      },
+      "residual_scales": {
+        "shard": 3,
+        "offset": 32737280,
+        "bytes": 65536
+      }
+    },
+    "blocks.14.cross_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 3,
+        "offset": 32802816,
+        "bytes": 2048
+      }
+    },
+    "blocks.14.cross_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 3,
+        "offset": 32804864,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 3,
+        "offset": 33853440,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 3,
+        "offset": 33886208,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 3,
+        "offset": 34410496,
+        "bytes": 32768
+      }
+    },
+    "blocks.14.cross_attn.to_q.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 3,
+        "offset": 34443264,
+        "bytes": 2048
+      }
+    },
+    "blocks.14.cross_attn.to_q.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 3,
+        "offset": 34445312,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 3,
+        "offset": 35493888,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 3,
+        "offset": 35526656,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 3,
+        "offset": 36050944,
+        "bytes": 32768
+      }
+    },
+    "blocks.14.mlp.mlp.0.bias": {
+      "shape": [
+        4096
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 3,
+        "offset": 36083712,
+        "bytes": 8192
+      }
+    },
+    "blocks.14.mlp.mlp.0.weight": {
+      "shape": [
+        4096,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 4096,
+      "columns": 1024,
+      "padded_rows": 4096,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 3,
+        "offset": 36091904,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 3,
+        "offset": 40286208,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 3,
+        "offset": 40417280,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 3,
+        "offset": 42514432,
+        "bytes": 131072
+      }
+    },
+    "blocks.14.mlp.mlp.2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 3,
+        "offset": 42645504,
+        "bytes": 2048
+      }
+    },
+    "blocks.14.mlp.mlp.2.weight": {
+      "shape": [
+        1024,
+        4096
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 4096,
+      "padded_rows": 1024,
+      "padded_columns": 4096,
+      "base_data": {
+        "shard": 3,
+        "offset": 42647552,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 3,
+        "offset": 46841856,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 3,
+        "offset": 46972928,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 3,
+        "offset": 49070080,
+        "bytes": 131072
+      }
+    },
+    "blocks.14.norm2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 3,
+        "offset": 49201152,
+        "bytes": 2048
+      }
+    },
+    "blocks.14.norm2.weight": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 3,
+        "offset": 49203200,
+        "bytes": 2048
+      }
+    },
+    "blocks.14.self_attn.k_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 3,
+        "offset": 49205248,
+        "bytes": 2048
+      }
+    },
+    "blocks.14.self_attn.q_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 3,
+        "offset": 49207296,
+        "bytes": 2048
+      }
+    },
+    "blocks.14.self_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 3,
+        "offset": 49209344,
+        "bytes": 2048
+      }
+    },
+    "blocks.14.self_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 3,
+        "offset": 49211392,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 3,
+        "offset": 50259968,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 3,
+        "offset": 50292736,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 3,
+        "offset": 50817024,
+        "bytes": 32768
+      }
+    },
+    "blocks.14.self_attn.to_qkv.bias": {
+      "shape": [
+        3072
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 3,
+        "offset": 50849792,
+        "bytes": 6144
+      }
+    },
+    "blocks.14.self_attn.to_qkv.weight": {
+      "shape": [
+        3072,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 3072,
+      "columns": 1024,
+      "padded_rows": 3072,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 3,
+        "offset": 50855936,
+        "bytes": 3145728
+      },
+      "base_scales": {
+        "shard": 3,
+        "offset": 54001664,
+        "bytes": 98304
+      },
+      "residual_data": {
+        "shard": 3,
+        "offset": 54099968,
+        "bytes": 1572864
+      },
+      "residual_scales": {
+        "shard": 3,
+        "offset": 55672832,
+        "bytes": 98304
+      }
+    },
+    "blocks.15.adaLN_modulation.1.bias": {
+      "shape": [
+        6144
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 3,
+        "offset": 55771136,
+        "bytes": 12288
+      }
+    },
+    "blocks.15.adaLN_modulation.1.weight": {
+      "shape": [
+        6144,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 6144,
+      "columns": 1024,
+      "padded_rows": 6144,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 3,
+        "offset": 55783424,
+        "bytes": 6291456
+      },
+      "base_scales": {
+        "shard": 3,
+        "offset": 62074880,
+        "bytes": 196608
+      },
+      "residual_data": {
+        "shard": 3,
+        "offset": 62271488,
+        "bytes": 3145728
+      },
+      "residual_scales": {
+        "shard": 3,
+        "offset": 65417216,
+        "bytes": 196608
+      }
+    },
+    "blocks.15.cross_attn.to_kv.bias": {
+      "shape": [
+        2048
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 3,
+        "offset": 65613824,
+        "bytes": 4096
+      }
+    },
+    "blocks.15.cross_attn.to_kv.weight": {
+      "shape": [
+        2048,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 2048,
+      "columns": 1024,
+      "padded_rows": 2048,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 4,
+        "offset": 0,
+        "bytes": 2097152
+      },
+      "base_scales": {
+        "shard": 4,
+        "offset": 2097152,
+        "bytes": 65536
+      },
+      "residual_data": {
+        "shard": 4,
+        "offset": 2162688,
+        "bytes": 1048576
+      },
+      "residual_scales": {
+        "shard": 4,
+        "offset": 3211264,
+        "bytes": 65536
+      }
+    },
+    "blocks.15.cross_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 4,
+        "offset": 3276800,
+        "bytes": 2048
+      }
+    },
+    "blocks.15.cross_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 4,
+        "offset": 3278848,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 4,
+        "offset": 4327424,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 4,
+        "offset": 4360192,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 4,
+        "offset": 4884480,
+        "bytes": 32768
+      }
+    },
+    "blocks.15.cross_attn.to_q.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 4,
+        "offset": 4917248,
+        "bytes": 2048
+      }
+    },
+    "blocks.15.cross_attn.to_q.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 4,
+        "offset": 4919296,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 4,
+        "offset": 5967872,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 4,
+        "offset": 6000640,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 4,
+        "offset": 6524928,
+        "bytes": 32768
+      }
+    },
+    "blocks.15.mlp.mlp.0.bias": {
+      "shape": [
+        4096
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 4,
+        "offset": 6557696,
+        "bytes": 8192
+      }
+    },
+    "blocks.15.mlp.mlp.0.weight": {
+      "shape": [
+        4096,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 4096,
+      "columns": 1024,
+      "padded_rows": 4096,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 4,
+        "offset": 6565888,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 4,
+        "offset": 10760192,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 4,
+        "offset": 10891264,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 4,
+        "offset": 12988416,
+        "bytes": 131072
+      }
+    },
+    "blocks.15.mlp.mlp.2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 4,
+        "offset": 13119488,
+        "bytes": 2048
+      }
+    },
+    "blocks.15.mlp.mlp.2.weight": {
+      "shape": [
+        1024,
+        4096
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 4096,
+      "padded_rows": 1024,
+      "padded_columns": 4096,
+      "base_data": {
+        "shard": 4,
+        "offset": 13121536,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 4,
+        "offset": 17315840,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 4,
+        "offset": 17446912,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 4,
+        "offset": 19544064,
+        "bytes": 131072
+      }
+    },
+    "blocks.15.norm2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 4,
+        "offset": 19675136,
+        "bytes": 2048
+      }
+    },
+    "blocks.15.norm2.weight": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 4,
+        "offset": 19677184,
+        "bytes": 2048
+      }
+    },
+    "blocks.15.self_attn.k_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 4,
+        "offset": 19679232,
+        "bytes": 2048
+      }
+    },
+    "blocks.15.self_attn.q_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 4,
+        "offset": 19681280,
+        "bytes": 2048
+      }
+    },
+    "blocks.15.self_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 4,
+        "offset": 19683328,
+        "bytes": 2048
+      }
+    },
+    "blocks.15.self_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 4,
+        "offset": 19685376,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 4,
+        "offset": 20733952,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 4,
+        "offset": 20766720,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 4,
+        "offset": 21291008,
+        "bytes": 32768
+      }
+    },
+    "blocks.15.self_attn.to_qkv.bias": {
+      "shape": [
+        3072
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 4,
+        "offset": 21323776,
+        "bytes": 6144
+      }
+    },
+    "blocks.15.self_attn.to_qkv.weight": {
+      "shape": [
+        3072,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 3072,
+      "columns": 1024,
+      "padded_rows": 3072,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 4,
+        "offset": 21329920,
+        "bytes": 3145728
+      },
+      "base_scales": {
+        "shard": 4,
+        "offset": 24475648,
+        "bytes": 98304
+      },
+      "residual_data": {
+        "shard": 4,
+        "offset": 24573952,
+        "bytes": 1572864
+      },
+      "residual_scales": {
+        "shard": 4,
+        "offset": 26146816,
+        "bytes": 98304
+      }
+    },
+    "blocks.16.adaLN_modulation.1.bias": {
+      "shape": [
+        6144
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 4,
+        "offset": 26245120,
+        "bytes": 12288
+      }
+    },
+    "blocks.16.adaLN_modulation.1.weight": {
+      "shape": [
+        6144,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 6144,
+      "columns": 1024,
+      "padded_rows": 6144,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 4,
+        "offset": 26257408,
+        "bytes": 6291456
+      },
+      "base_scales": {
+        "shard": 4,
+        "offset": 32548864,
+        "bytes": 196608
+      },
+      "residual_data": {
+        "shard": 4,
+        "offset": 32745472,
+        "bytes": 3145728
+      },
+      "residual_scales": {
+        "shard": 4,
+        "offset": 35891200,
+        "bytes": 196608
+      }
+    },
+    "blocks.16.cross_attn.to_kv.bias": {
+      "shape": [
+        2048
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 4,
+        "offset": 36087808,
+        "bytes": 4096
+      }
+    },
+    "blocks.16.cross_attn.to_kv.weight": {
+      "shape": [
+        2048,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 2048,
+      "columns": 1024,
+      "padded_rows": 2048,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 4,
+        "offset": 36091904,
+        "bytes": 2097152
+      },
+      "base_scales": {
+        "shard": 4,
+        "offset": 38189056,
+        "bytes": 65536
+      },
+      "residual_data": {
+        "shard": 4,
+        "offset": 38254592,
+        "bytes": 1048576
+      },
+      "residual_scales": {
+        "shard": 4,
+        "offset": 39303168,
+        "bytes": 65536
+      }
+    },
+    "blocks.16.cross_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 4,
+        "offset": 39368704,
+        "bytes": 2048
+      }
+    },
+    "blocks.16.cross_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 4,
+        "offset": 39370752,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 4,
+        "offset": 40419328,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 4,
+        "offset": 40452096,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 4,
+        "offset": 40976384,
+        "bytes": 32768
+      }
+    },
+    "blocks.16.cross_attn.to_q.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 4,
+        "offset": 41009152,
+        "bytes": 2048
+      }
+    },
+    "blocks.16.cross_attn.to_q.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 4,
+        "offset": 41011200,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 4,
+        "offset": 42059776,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 4,
+        "offset": 42092544,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 4,
+        "offset": 42616832,
+        "bytes": 32768
+      }
+    },
+    "blocks.16.mlp.mlp.0.bias": {
+      "shape": [
+        4096
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 4,
+        "offset": 42649600,
+        "bytes": 8192
+      }
+    },
+    "blocks.16.mlp.mlp.0.weight": {
+      "shape": [
+        4096,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 4096,
+      "columns": 1024,
+      "padded_rows": 4096,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 4,
+        "offset": 42657792,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 4,
+        "offset": 46852096,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 4,
+        "offset": 46983168,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 4,
+        "offset": 49080320,
+        "bytes": 131072
+      }
+    },
+    "blocks.16.mlp.mlp.2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 4,
+        "offset": 49211392,
+        "bytes": 2048
+      }
+    },
+    "blocks.16.mlp.mlp.2.weight": {
+      "shape": [
+        1024,
+        4096
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 4096,
+      "padded_rows": 1024,
+      "padded_columns": 4096,
+      "base_data": {
+        "shard": 4,
+        "offset": 49213440,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 4,
+        "offset": 53407744,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 4,
+        "offset": 53538816,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 4,
+        "offset": 55635968,
+        "bytes": 131072
+      }
+    },
+    "blocks.16.norm2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 4,
+        "offset": 55767040,
+        "bytes": 2048
+      }
+    },
+    "blocks.16.norm2.weight": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 4,
+        "offset": 55769088,
+        "bytes": 2048
+      }
+    },
+    "blocks.16.self_attn.k_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 4,
+        "offset": 55771136,
+        "bytes": 2048
+      }
+    },
+    "blocks.16.self_attn.q_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 4,
+        "offset": 55773184,
+        "bytes": 2048
+      }
+    },
+    "blocks.16.self_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 4,
+        "offset": 55775232,
+        "bytes": 2048
+      }
+    },
+    "blocks.16.self_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 4,
+        "offset": 55777280,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 4,
+        "offset": 56825856,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 4,
+        "offset": 56858624,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 4,
+        "offset": 57382912,
+        "bytes": 32768
+      }
+    },
+    "blocks.16.self_attn.to_qkv.bias": {
+      "shape": [
+        3072
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 4,
+        "offset": 57415680,
+        "bytes": 6144
+      }
+    },
+    "blocks.16.self_attn.to_qkv.weight": {
+      "shape": [
+        3072,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 3072,
+      "columns": 1024,
+      "padded_rows": 3072,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 4,
+        "offset": 57421824,
+        "bytes": 3145728
+      },
+      "base_scales": {
+        "shard": 4,
+        "offset": 60567552,
+        "bytes": 98304
+      },
+      "residual_data": {
+        "shard": 4,
+        "offset": 60665856,
+        "bytes": 1572864
+      },
+      "residual_scales": {
+        "shard": 4,
+        "offset": 62238720,
+        "bytes": 98304
+      }
+    },
+    "blocks.17.adaLN_modulation.1.bias": {
+      "shape": [
+        6144
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 4,
+        "offset": 62337024,
+        "bytes": 12288
+      }
+    },
+    "blocks.17.adaLN_modulation.1.weight": {
+      "shape": [
+        6144,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 6144,
+      "columns": 1024,
+      "padded_rows": 6144,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 5,
+        "offset": 0,
+        "bytes": 6291456
+      },
+      "base_scales": {
+        "shard": 5,
+        "offset": 6291456,
+        "bytes": 196608
+      },
+      "residual_data": {
+        "shard": 5,
+        "offset": 6488064,
+        "bytes": 3145728
+      },
+      "residual_scales": {
+        "shard": 5,
+        "offset": 9633792,
+        "bytes": 196608
+      }
+    },
+    "blocks.17.cross_attn.to_kv.bias": {
+      "shape": [
+        2048
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 5,
+        "offset": 9830400,
+        "bytes": 4096
+      }
+    },
+    "blocks.17.cross_attn.to_kv.weight": {
+      "shape": [
+        2048,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 2048,
+      "columns": 1024,
+      "padded_rows": 2048,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 5,
+        "offset": 9834496,
+        "bytes": 2097152
+      },
+      "base_scales": {
+        "shard": 5,
+        "offset": 11931648,
+        "bytes": 65536
+      },
+      "residual_data": {
+        "shard": 5,
+        "offset": 11997184,
+        "bytes": 1048576
+      },
+      "residual_scales": {
+        "shard": 5,
+        "offset": 13045760,
+        "bytes": 65536
+      }
+    },
+    "blocks.17.cross_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 5,
+        "offset": 13111296,
+        "bytes": 2048
+      }
+    },
+    "blocks.17.cross_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 5,
+        "offset": 13113344,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 5,
+        "offset": 14161920,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 5,
+        "offset": 14194688,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 5,
+        "offset": 14718976,
+        "bytes": 32768
+      }
+    },
+    "blocks.17.cross_attn.to_q.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 5,
+        "offset": 14751744,
+        "bytes": 2048
+      }
+    },
+    "blocks.17.cross_attn.to_q.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 5,
+        "offset": 14753792,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 5,
+        "offset": 15802368,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 5,
+        "offset": 15835136,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 5,
+        "offset": 16359424,
+        "bytes": 32768
+      }
+    },
+    "blocks.17.mlp.mlp.0.bias": {
+      "shape": [
+        4096
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 5,
+        "offset": 16392192,
+        "bytes": 8192
+      }
+    },
+    "blocks.17.mlp.mlp.0.weight": {
+      "shape": [
+        4096,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 4096,
+      "columns": 1024,
+      "padded_rows": 4096,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 5,
+        "offset": 16400384,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 5,
+        "offset": 20594688,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 5,
+        "offset": 20725760,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 5,
+        "offset": 22822912,
+        "bytes": 131072
+      }
+    },
+    "blocks.17.mlp.mlp.2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 5,
+        "offset": 22953984,
+        "bytes": 2048
+      }
+    },
+    "blocks.17.mlp.mlp.2.weight": {
+      "shape": [
+        1024,
+        4096
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 4096,
+      "padded_rows": 1024,
+      "padded_columns": 4096,
+      "base_data": {
+        "shard": 5,
+        "offset": 22956032,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 5,
+        "offset": 27150336,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 5,
+        "offset": 27281408,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 5,
+        "offset": 29378560,
+        "bytes": 131072
+      }
+    },
+    "blocks.17.norm2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 5,
+        "offset": 29509632,
+        "bytes": 2048
+      }
+    },
+    "blocks.17.norm2.weight": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 5,
+        "offset": 29511680,
+        "bytes": 2048
+      }
+    },
+    "blocks.17.self_attn.k_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 5,
+        "offset": 29513728,
+        "bytes": 2048
+      }
+    },
+    "blocks.17.self_attn.q_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 5,
+        "offset": 29515776,
+        "bytes": 2048
+      }
+    },
+    "blocks.17.self_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 5,
+        "offset": 29517824,
+        "bytes": 2048
+      }
+    },
+    "blocks.17.self_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 5,
+        "offset": 29519872,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 5,
+        "offset": 30568448,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 5,
+        "offset": 30601216,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 5,
+        "offset": 31125504,
+        "bytes": 32768
+      }
+    },
+    "blocks.17.self_attn.to_qkv.bias": {
+      "shape": [
+        3072
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 5,
+        "offset": 31158272,
+        "bytes": 6144
+      }
+    },
+    "blocks.17.self_attn.to_qkv.weight": {
+      "shape": [
+        3072,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 3072,
+      "columns": 1024,
+      "padded_rows": 3072,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 5,
+        "offset": 31164416,
+        "bytes": 3145728
+      },
+      "base_scales": {
+        "shard": 5,
+        "offset": 34310144,
+        "bytes": 98304
+      },
+      "residual_data": {
+        "shard": 5,
+        "offset": 34408448,
+        "bytes": 1572864
+      },
+      "residual_scales": {
+        "shard": 5,
+        "offset": 35981312,
+        "bytes": 98304
+      }
+    },
+    "blocks.18.adaLN_modulation.1.bias": {
+      "shape": [
+        6144
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 5,
+        "offset": 36079616,
+        "bytes": 12288
+      }
+    },
+    "blocks.18.adaLN_modulation.1.weight": {
+      "shape": [
+        6144,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 6144,
+      "columns": 1024,
+      "padded_rows": 6144,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 5,
+        "offset": 36091904,
+        "bytes": 6291456
+      },
+      "base_scales": {
+        "shard": 5,
+        "offset": 42383360,
+        "bytes": 196608
+      },
+      "residual_data": {
+        "shard": 5,
+        "offset": 42579968,
+        "bytes": 3145728
+      },
+      "residual_scales": {
+        "shard": 5,
+        "offset": 45725696,
+        "bytes": 196608
+      }
+    },
+    "blocks.18.cross_attn.to_kv.bias": {
+      "shape": [
+        2048
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 5,
+        "offset": 45922304,
+        "bytes": 4096
+      }
+    },
+    "blocks.18.cross_attn.to_kv.weight": {
+      "shape": [
+        2048,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 2048,
+      "columns": 1024,
+      "padded_rows": 2048,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 5,
+        "offset": 45926400,
+        "bytes": 2097152
+      },
+      "base_scales": {
+        "shard": 5,
+        "offset": 48023552,
+        "bytes": 65536
+      },
+      "residual_data": {
+        "shard": 5,
+        "offset": 48089088,
+        "bytes": 1048576
+      },
+      "residual_scales": {
+        "shard": 5,
+        "offset": 49137664,
+        "bytes": 65536
+      }
+    },
+    "blocks.18.cross_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 5,
+        "offset": 49203200,
+        "bytes": 2048
+      }
+    },
+    "blocks.18.cross_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 5,
+        "offset": 49205248,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 5,
+        "offset": 50253824,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 5,
+        "offset": 50286592,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 5,
+        "offset": 50810880,
+        "bytes": 32768
+      }
+    },
+    "blocks.18.cross_attn.to_q.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 5,
+        "offset": 50843648,
+        "bytes": 2048
+      }
+    },
+    "blocks.18.cross_attn.to_q.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 5,
+        "offset": 50845696,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 5,
+        "offset": 51894272,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 5,
+        "offset": 51927040,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 5,
+        "offset": 52451328,
+        "bytes": 32768
+      }
+    },
+    "blocks.18.mlp.mlp.0.bias": {
+      "shape": [
+        4096
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 5,
+        "offset": 52484096,
+        "bytes": 8192
+      }
+    },
+    "blocks.18.mlp.mlp.0.weight": {
+      "shape": [
+        4096,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 4096,
+      "columns": 1024,
+      "padded_rows": 4096,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 5,
+        "offset": 52492288,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 5,
+        "offset": 56686592,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 5,
+        "offset": 56817664,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 5,
+        "offset": 58914816,
+        "bytes": 131072
+      }
+    },
+    "blocks.18.mlp.mlp.2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 5,
+        "offset": 59045888,
+        "bytes": 2048
+      }
+    },
+    "blocks.18.mlp.mlp.2.weight": {
+      "shape": [
+        1024,
+        4096
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 4096,
+      "padded_rows": 1024,
+      "padded_columns": 4096,
+      "base_data": {
+        "shard": 5,
+        "offset": 59047936,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 5,
+        "offset": 63242240,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 5,
+        "offset": 63373312,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 5,
+        "offset": 65470464,
+        "bytes": 131072
+      }
+    },
+    "blocks.18.norm2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 5,
+        "offset": 65601536,
+        "bytes": 2048
+      }
+    },
+    "blocks.18.norm2.weight": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 5,
+        "offset": 65603584,
+        "bytes": 2048
+      }
+    },
+    "blocks.18.self_attn.k_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 5,
+        "offset": 65605632,
+        "bytes": 2048
+      }
+    },
+    "blocks.18.self_attn.q_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 5,
+        "offset": 65607680,
+        "bytes": 2048
+      }
+    },
+    "blocks.18.self_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 5,
+        "offset": 65609728,
+        "bytes": 2048
+      }
+    },
+    "blocks.18.self_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 5,
+        "offset": 65611776,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 5,
+        "offset": 66660352,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 6,
+        "offset": 0,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 6,
+        "offset": 524288,
+        "bytes": 32768
+      }
+    },
+    "blocks.18.self_attn.to_qkv.bias": {
+      "shape": [
+        3072
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 6,
+        "offset": 557056,
+        "bytes": 6144
+      }
+    },
+    "blocks.18.self_attn.to_qkv.weight": {
+      "shape": [
+        3072,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 3072,
+      "columns": 1024,
+      "padded_rows": 3072,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 6,
+        "offset": 563200,
+        "bytes": 3145728
+      },
+      "base_scales": {
+        "shard": 6,
+        "offset": 3708928,
+        "bytes": 98304
+      },
+      "residual_data": {
+        "shard": 6,
+        "offset": 3807232,
+        "bytes": 1572864
+      },
+      "residual_scales": {
+        "shard": 6,
+        "offset": 5380096,
+        "bytes": 98304
+      }
+    },
+    "blocks.19.adaLN_modulation.1.bias": {
+      "shape": [
+        6144
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 6,
+        "offset": 5478400,
+        "bytes": 12288
+      }
+    },
+    "blocks.19.adaLN_modulation.1.weight": {
+      "shape": [
+        6144,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 6144,
+      "columns": 1024,
+      "padded_rows": 6144,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 6,
+        "offset": 5490688,
+        "bytes": 6291456
+      },
+      "base_scales": {
+        "shard": 6,
+        "offset": 11782144,
+        "bytes": 196608
+      },
+      "residual_data": {
+        "shard": 6,
+        "offset": 11978752,
+        "bytes": 3145728
+      },
+      "residual_scales": {
+        "shard": 6,
+        "offset": 15124480,
+        "bytes": 196608
+      }
+    },
+    "blocks.19.cross_attn.to_kv.bias": {
+      "shape": [
+        2048
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 6,
+        "offset": 15321088,
+        "bytes": 4096
+      }
+    },
+    "blocks.19.cross_attn.to_kv.weight": {
+      "shape": [
+        2048,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 2048,
+      "columns": 1024,
+      "padded_rows": 2048,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 6,
+        "offset": 15325184,
+        "bytes": 2097152
+      },
+      "base_scales": {
+        "shard": 6,
+        "offset": 17422336,
+        "bytes": 65536
+      },
+      "residual_data": {
+        "shard": 6,
+        "offset": 17487872,
+        "bytes": 1048576
+      },
+      "residual_scales": {
+        "shard": 6,
+        "offset": 18536448,
+        "bytes": 65536
+      }
+    },
+    "blocks.19.cross_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 6,
+        "offset": 18601984,
+        "bytes": 2048
+      }
+    },
+    "blocks.19.cross_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 6,
+        "offset": 18604032,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 6,
+        "offset": 19652608,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 6,
+        "offset": 19685376,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 6,
+        "offset": 20209664,
+        "bytes": 32768
+      }
+    },
+    "blocks.19.cross_attn.to_q.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 6,
+        "offset": 20242432,
+        "bytes": 2048
+      }
+    },
+    "blocks.19.cross_attn.to_q.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 6,
+        "offset": 20244480,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 6,
+        "offset": 21293056,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 6,
+        "offset": 21325824,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 6,
+        "offset": 21850112,
+        "bytes": 32768
+      }
+    },
+    "blocks.19.mlp.mlp.0.bias": {
+      "shape": [
+        4096
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 6,
+        "offset": 21882880,
+        "bytes": 8192
+      }
+    },
+    "blocks.19.mlp.mlp.0.weight": {
+      "shape": [
+        4096,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 4096,
+      "columns": 1024,
+      "padded_rows": 4096,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 6,
+        "offset": 21891072,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 6,
+        "offset": 26085376,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 6,
+        "offset": 26216448,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 6,
+        "offset": 28313600,
+        "bytes": 131072
+      }
+    },
+    "blocks.19.mlp.mlp.2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 6,
+        "offset": 28444672,
+        "bytes": 2048
+      }
+    },
+    "blocks.19.mlp.mlp.2.weight": {
+      "shape": [
+        1024,
+        4096
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 4096,
+      "padded_rows": 1024,
+      "padded_columns": 4096,
+      "base_data": {
+        "shard": 6,
+        "offset": 28446720,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 6,
+        "offset": 32641024,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 6,
+        "offset": 32772096,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 6,
+        "offset": 34869248,
+        "bytes": 131072
+      }
+    },
+    "blocks.19.norm2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 6,
+        "offset": 35000320,
+        "bytes": 2048
+      }
+    },
+    "blocks.19.norm2.weight": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 6,
+        "offset": 35002368,
+        "bytes": 2048
+      }
+    },
+    "blocks.19.self_attn.k_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 6,
+        "offset": 35004416,
+        "bytes": 2048
+      }
+    },
+    "blocks.19.self_attn.q_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 6,
+        "offset": 35006464,
+        "bytes": 2048
+      }
+    },
+    "blocks.19.self_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 6,
+        "offset": 35008512,
+        "bytes": 2048
+      }
+    },
+    "blocks.19.self_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 6,
+        "offset": 35010560,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 6,
+        "offset": 36059136,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 6,
+        "offset": 36091904,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 6,
+        "offset": 36616192,
+        "bytes": 32768
+      }
+    },
+    "blocks.19.self_attn.to_qkv.bias": {
+      "shape": [
+        3072
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 6,
+        "offset": 36648960,
+        "bytes": 6144
+      }
+    },
+    "blocks.19.self_attn.to_qkv.weight": {
+      "shape": [
+        3072,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 3072,
+      "columns": 1024,
+      "padded_rows": 3072,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 6,
+        "offset": 36655104,
+        "bytes": 3145728
+      },
+      "base_scales": {
+        "shard": 6,
+        "offset": 39800832,
+        "bytes": 98304
+      },
+      "residual_data": {
+        "shard": 6,
+        "offset": 39899136,
+        "bytes": 1572864
+      },
+      "residual_scales": {
+        "shard": 6,
+        "offset": 41472000,
+        "bytes": 98304
+      }
+    },
+    "blocks.2.adaLN_modulation.1.bias": {
+      "shape": [
+        6144
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 6,
+        "offset": 41570304,
+        "bytes": 12288
+      }
+    },
+    "blocks.2.adaLN_modulation.1.weight": {
+      "shape": [
+        6144,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 6144,
+      "columns": 1024,
+      "padded_rows": 6144,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 6,
+        "offset": 41582592,
+        "bytes": 6291456
+      },
+      "base_scales": {
+        "shard": 6,
+        "offset": 47874048,
+        "bytes": 196608
+      },
+      "residual_data": {
+        "shard": 6,
+        "offset": 48070656,
+        "bytes": 3145728
+      },
+      "residual_scales": {
+        "shard": 6,
+        "offset": 51216384,
+        "bytes": 196608
+      }
+    },
+    "blocks.2.cross_attn.to_kv.bias": {
+      "shape": [
+        2048
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 6,
+        "offset": 51412992,
+        "bytes": 4096
+      }
+    },
+    "blocks.2.cross_attn.to_kv.weight": {
+      "shape": [
+        2048,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 2048,
+      "columns": 1024,
+      "padded_rows": 2048,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 6,
+        "offset": 51417088,
+        "bytes": 2097152
+      },
+      "base_scales": {
+        "shard": 6,
+        "offset": 53514240,
+        "bytes": 65536
+      },
+      "residual_data": {
+        "shard": 6,
+        "offset": 53579776,
+        "bytes": 1048576
+      },
+      "residual_scales": {
+        "shard": 6,
+        "offset": 54628352,
+        "bytes": 65536
+      }
+    },
+    "blocks.2.cross_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 6,
+        "offset": 54693888,
+        "bytes": 2048
+      }
+    },
+    "blocks.2.cross_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 6,
+        "offset": 54695936,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 6,
+        "offset": 55744512,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 6,
+        "offset": 55777280,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 6,
+        "offset": 56301568,
+        "bytes": 32768
+      }
+    },
+    "blocks.2.cross_attn.to_q.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 6,
+        "offset": 56334336,
+        "bytes": 2048
+      }
+    },
+    "blocks.2.cross_attn.to_q.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 6,
+        "offset": 56336384,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 6,
+        "offset": 57384960,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 6,
+        "offset": 57417728,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 6,
+        "offset": 57942016,
+        "bytes": 32768
+      }
+    },
+    "blocks.2.mlp.mlp.0.bias": {
+      "shape": [
+        4096
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 6,
+        "offset": 57974784,
+        "bytes": 8192
+      }
+    },
+    "blocks.2.mlp.mlp.0.weight": {
+      "shape": [
+        4096,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 4096,
+      "columns": 1024,
+      "padded_rows": 4096,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 6,
+        "offset": 57982976,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 6,
+        "offset": 62177280,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 6,
+        "offset": 62308352,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 6,
+        "offset": 64405504,
+        "bytes": 131072
+      }
+    },
+    "blocks.2.mlp.mlp.2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 6,
+        "offset": 64536576,
+        "bytes": 2048
+      }
+    },
+    "blocks.2.mlp.mlp.2.weight": {
+      "shape": [
+        1024,
+        4096
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 4096,
+      "padded_rows": 1024,
+      "padded_columns": 4096,
+      "base_data": {
+        "shard": 7,
+        "offset": 0,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 7,
+        "offset": 4194304,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 7,
+        "offset": 4325376,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 7,
+        "offset": 6422528,
+        "bytes": 131072
+      }
+    },
+    "blocks.2.norm2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 7,
+        "offset": 6553600,
+        "bytes": 2048
+      }
+    },
+    "blocks.2.norm2.weight": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 7,
+        "offset": 6555648,
+        "bytes": 2048
+      }
+    },
+    "blocks.2.self_attn.k_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 7,
+        "offset": 6557696,
+        "bytes": 2048
+      }
+    },
+    "blocks.2.self_attn.q_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 7,
+        "offset": 6559744,
+        "bytes": 2048
+      }
+    },
+    "blocks.2.self_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 7,
+        "offset": 6561792,
+        "bytes": 2048
+      }
+    },
+    "blocks.2.self_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 7,
+        "offset": 6563840,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 7,
+        "offset": 7612416,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 7,
+        "offset": 7645184,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 7,
+        "offset": 8169472,
+        "bytes": 32768
+      }
+    },
+    "blocks.2.self_attn.to_qkv.bias": {
+      "shape": [
+        3072
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 7,
+        "offset": 8202240,
+        "bytes": 6144
+      }
+    },
+    "blocks.2.self_attn.to_qkv.weight": {
+      "shape": [
+        3072,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 3072,
+      "columns": 1024,
+      "padded_rows": 3072,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 7,
+        "offset": 8208384,
+        "bytes": 3145728
+      },
+      "base_scales": {
+        "shard": 7,
+        "offset": 11354112,
+        "bytes": 98304
+      },
+      "residual_data": {
+        "shard": 7,
+        "offset": 11452416,
+        "bytes": 1572864
+      },
+      "residual_scales": {
+        "shard": 7,
+        "offset": 13025280,
+        "bytes": 98304
+      }
+    },
+    "blocks.20.adaLN_modulation.1.bias": {
+      "shape": [
+        6144
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 7,
+        "offset": 13123584,
+        "bytes": 12288
+      }
+    },
+    "blocks.20.adaLN_modulation.1.weight": {
+      "shape": [
+        6144,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 6144,
+      "columns": 1024,
+      "padded_rows": 6144,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 7,
+        "offset": 13135872,
+        "bytes": 6291456
+      },
+      "base_scales": {
+        "shard": 7,
+        "offset": 19427328,
+        "bytes": 196608
+      },
+      "residual_data": {
+        "shard": 7,
+        "offset": 19623936,
+        "bytes": 3145728
+      },
+      "residual_scales": {
+        "shard": 7,
+        "offset": 22769664,
+        "bytes": 196608
+      }
+    },
+    "blocks.20.cross_attn.to_kv.bias": {
+      "shape": [
+        2048
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 7,
+        "offset": 22966272,
+        "bytes": 4096
+      }
+    },
+    "blocks.20.cross_attn.to_kv.weight": {
+      "shape": [
+        2048,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 2048,
+      "columns": 1024,
+      "padded_rows": 2048,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 7,
+        "offset": 22970368,
+        "bytes": 2097152
+      },
+      "base_scales": {
+        "shard": 7,
+        "offset": 25067520,
+        "bytes": 65536
+      },
+      "residual_data": {
+        "shard": 7,
+        "offset": 25133056,
+        "bytes": 1048576
+      },
+      "residual_scales": {
+        "shard": 7,
+        "offset": 26181632,
+        "bytes": 65536
+      }
+    },
+    "blocks.20.cross_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 7,
+        "offset": 26247168,
+        "bytes": 2048
+      }
+    },
+    "blocks.20.cross_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 7,
+        "offset": 26249216,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 7,
+        "offset": 27297792,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 7,
+        "offset": 27330560,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 7,
+        "offset": 27854848,
+        "bytes": 32768
+      }
+    },
+    "blocks.20.cross_attn.to_q.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 7,
+        "offset": 27887616,
+        "bytes": 2048
+      }
+    },
+    "blocks.20.cross_attn.to_q.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 7,
+        "offset": 27889664,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 7,
+        "offset": 28938240,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 7,
+        "offset": 28971008,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 7,
+        "offset": 29495296,
+        "bytes": 32768
+      }
+    },
+    "blocks.20.mlp.mlp.0.bias": {
+      "shape": [
+        4096
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 7,
+        "offset": 29528064,
+        "bytes": 8192
+      }
+    },
+    "blocks.20.mlp.mlp.0.weight": {
+      "shape": [
+        4096,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 4096,
+      "columns": 1024,
+      "padded_rows": 4096,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 7,
+        "offset": 29536256,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 7,
+        "offset": 33730560,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 7,
+        "offset": 33861632,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 7,
+        "offset": 35958784,
+        "bytes": 131072
+      }
+    },
+    "blocks.20.mlp.mlp.2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 7,
+        "offset": 36089856,
+        "bytes": 2048
+      }
+    },
+    "blocks.20.mlp.mlp.2.weight": {
+      "shape": [
+        1024,
+        4096
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 4096,
+      "padded_rows": 1024,
+      "padded_columns": 4096,
+      "base_data": {
+        "shard": 7,
+        "offset": 36091904,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 7,
+        "offset": 40286208,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 7,
+        "offset": 40417280,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 7,
+        "offset": 42514432,
+        "bytes": 131072
+      }
+    },
+    "blocks.20.norm2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 7,
+        "offset": 42645504,
+        "bytes": 2048
+      }
+    },
+    "blocks.20.norm2.weight": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 7,
+        "offset": 42647552,
+        "bytes": 2048
+      }
+    },
+    "blocks.20.self_attn.k_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 7,
+        "offset": 42649600,
+        "bytes": 2048
+      }
+    },
+    "blocks.20.self_attn.q_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 7,
+        "offset": 42651648,
+        "bytes": 2048
+      }
+    },
+    "blocks.20.self_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 7,
+        "offset": 42653696,
+        "bytes": 2048
+      }
+    },
+    "blocks.20.self_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 7,
+        "offset": 42655744,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 7,
+        "offset": 43704320,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 7,
+        "offset": 43737088,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 7,
+        "offset": 44261376,
+        "bytes": 32768
+      }
+    },
+    "blocks.20.self_attn.to_qkv.bias": {
+      "shape": [
+        3072
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 7,
+        "offset": 44294144,
+        "bytes": 6144
+      }
+    },
+    "blocks.20.self_attn.to_qkv.weight": {
+      "shape": [
+        3072,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 3072,
+      "columns": 1024,
+      "padded_rows": 3072,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 7,
+        "offset": 44300288,
+        "bytes": 3145728
+      },
+      "base_scales": {
+        "shard": 7,
+        "offset": 47446016,
+        "bytes": 98304
+      },
+      "residual_data": {
+        "shard": 7,
+        "offset": 47544320,
+        "bytes": 1572864
+      },
+      "residual_scales": {
+        "shard": 7,
+        "offset": 49117184,
+        "bytes": 98304
+      }
+    },
+    "blocks.21.adaLN_modulation.1.bias": {
+      "shape": [
+        6144
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 7,
+        "offset": 49215488,
+        "bytes": 12288
+      }
+    },
+    "blocks.21.adaLN_modulation.1.weight": {
+      "shape": [
+        6144,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 6144,
+      "columns": 1024,
+      "padded_rows": 6144,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 7,
+        "offset": 49227776,
+        "bytes": 6291456
+      },
+      "base_scales": {
+        "shard": 7,
+        "offset": 55519232,
+        "bytes": 196608
+      },
+      "residual_data": {
+        "shard": 7,
+        "offset": 55715840,
+        "bytes": 3145728
+      },
+      "residual_scales": {
+        "shard": 7,
+        "offset": 58861568,
+        "bytes": 196608
+      }
+    },
+    "blocks.21.cross_attn.to_kv.bias": {
+      "shape": [
+        2048
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 7,
+        "offset": 59058176,
+        "bytes": 4096
+      }
+    },
+    "blocks.21.cross_attn.to_kv.weight": {
+      "shape": [
+        2048,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 2048,
+      "columns": 1024,
+      "padded_rows": 2048,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 7,
+        "offset": 59062272,
+        "bytes": 2097152
+      },
+      "base_scales": {
+        "shard": 7,
+        "offset": 61159424,
+        "bytes": 65536
+      },
+      "residual_data": {
+        "shard": 7,
+        "offset": 61224960,
+        "bytes": 1048576
+      },
+      "residual_scales": {
+        "shard": 7,
+        "offset": 62273536,
+        "bytes": 65536
+      }
+    },
+    "blocks.21.cross_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 7,
+        "offset": 62339072,
+        "bytes": 2048
+      }
+    },
+    "blocks.21.cross_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 7,
+        "offset": 62341120,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 7,
+        "offset": 63389696,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 7,
+        "offset": 63422464,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 7,
+        "offset": 63946752,
+        "bytes": 32768
+      }
+    },
+    "blocks.21.cross_attn.to_q.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 7,
+        "offset": 63979520,
+        "bytes": 2048
+      }
+    },
+    "blocks.21.cross_attn.to_q.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 7,
+        "offset": 63981568,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 7,
+        "offset": 65030144,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 7,
+        "offset": 65062912,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 7,
+        "offset": 65587200,
+        "bytes": 32768
+      }
+    },
+    "blocks.21.mlp.mlp.0.bias": {
+      "shape": [
+        4096
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 7,
+        "offset": 65619968,
+        "bytes": 8192
+      }
+    },
+    "blocks.21.mlp.mlp.0.weight": {
+      "shape": [
+        4096,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 4096,
+      "columns": 1024,
+      "padded_rows": 4096,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 8,
+        "offset": 0,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 8,
+        "offset": 4194304,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 8,
+        "offset": 4325376,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 8,
+        "offset": 6422528,
+        "bytes": 131072
+      }
+    },
+    "blocks.21.mlp.mlp.2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 8,
+        "offset": 6553600,
+        "bytes": 2048
+      }
+    },
+    "blocks.21.mlp.mlp.2.weight": {
+      "shape": [
+        1024,
+        4096
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 4096,
+      "padded_rows": 1024,
+      "padded_columns": 4096,
+      "base_data": {
+        "shard": 8,
+        "offset": 6555648,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 8,
+        "offset": 10749952,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 8,
+        "offset": 10881024,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 8,
+        "offset": 12978176,
+        "bytes": 131072
+      }
+    },
+    "blocks.21.norm2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 8,
+        "offset": 13109248,
+        "bytes": 2048
+      }
+    },
+    "blocks.21.norm2.weight": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 8,
+        "offset": 13111296,
+        "bytes": 2048
+      }
+    },
+    "blocks.21.self_attn.k_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 8,
+        "offset": 13113344,
+        "bytes": 2048
+      }
+    },
+    "blocks.21.self_attn.q_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 8,
+        "offset": 13115392,
+        "bytes": 2048
+      }
+    },
+    "blocks.21.self_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 8,
+        "offset": 13117440,
+        "bytes": 2048
+      }
+    },
+    "blocks.21.self_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 8,
+        "offset": 13119488,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 8,
+        "offset": 14168064,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 8,
+        "offset": 14200832,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 8,
+        "offset": 14725120,
+        "bytes": 32768
+      }
+    },
+    "blocks.21.self_attn.to_qkv.bias": {
+      "shape": [
+        3072
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 8,
+        "offset": 14757888,
+        "bytes": 6144
+      }
+    },
+    "blocks.21.self_attn.to_qkv.weight": {
+      "shape": [
+        3072,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 3072,
+      "columns": 1024,
+      "padded_rows": 3072,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 8,
+        "offset": 14764032,
+        "bytes": 3145728
+      },
+      "base_scales": {
+        "shard": 8,
+        "offset": 17909760,
+        "bytes": 98304
+      },
+      "residual_data": {
+        "shard": 8,
+        "offset": 18008064,
+        "bytes": 1572864
+      },
+      "residual_scales": {
+        "shard": 8,
+        "offset": 19580928,
+        "bytes": 98304
+      }
+    },
+    "blocks.22.adaLN_modulation.1.bias": {
+      "shape": [
+        6144
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 8,
+        "offset": 19679232,
+        "bytes": 12288
+      }
+    },
+    "blocks.22.adaLN_modulation.1.weight": {
+      "shape": [
+        6144,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 6144,
+      "columns": 1024,
+      "padded_rows": 6144,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 8,
+        "offset": 19691520,
+        "bytes": 6291456
+      },
+      "base_scales": {
+        "shard": 8,
+        "offset": 25982976,
+        "bytes": 196608
+      },
+      "residual_data": {
+        "shard": 8,
+        "offset": 26179584,
+        "bytes": 3145728
+      },
+      "residual_scales": {
+        "shard": 8,
+        "offset": 29325312,
+        "bytes": 196608
+      }
+    },
+    "blocks.22.cross_attn.to_kv.bias": {
+      "shape": [
+        2048
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 8,
+        "offset": 29521920,
+        "bytes": 4096
+      }
+    },
+    "blocks.22.cross_attn.to_kv.weight": {
+      "shape": [
+        2048,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 2048,
+      "columns": 1024,
+      "padded_rows": 2048,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 8,
+        "offset": 29526016,
+        "bytes": 2097152
+      },
+      "base_scales": {
+        "shard": 8,
+        "offset": 31623168,
+        "bytes": 65536
+      },
+      "residual_data": {
+        "shard": 8,
+        "offset": 31688704,
+        "bytes": 1048576
+      },
+      "residual_scales": {
+        "shard": 8,
+        "offset": 32737280,
+        "bytes": 65536
+      }
+    },
+    "blocks.22.cross_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 8,
+        "offset": 32802816,
+        "bytes": 2048
+      }
+    },
+    "blocks.22.cross_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 8,
+        "offset": 32804864,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 8,
+        "offset": 33853440,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 8,
+        "offset": 33886208,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 8,
+        "offset": 34410496,
+        "bytes": 32768
+      }
+    },
+    "blocks.22.cross_attn.to_q.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 8,
+        "offset": 34443264,
+        "bytes": 2048
+      }
+    },
+    "blocks.22.cross_attn.to_q.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 8,
+        "offset": 34445312,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 8,
+        "offset": 35493888,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 8,
+        "offset": 35526656,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 8,
+        "offset": 36050944,
+        "bytes": 32768
+      }
+    },
+    "blocks.22.mlp.mlp.0.bias": {
+      "shape": [
+        4096
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 8,
+        "offset": 36083712,
+        "bytes": 8192
+      }
+    },
+    "blocks.22.mlp.mlp.0.weight": {
+      "shape": [
+        4096,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 4096,
+      "columns": 1024,
+      "padded_rows": 4096,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 8,
+        "offset": 36091904,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 8,
+        "offset": 40286208,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 8,
+        "offset": 40417280,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 8,
+        "offset": 42514432,
+        "bytes": 131072
+      }
+    },
+    "blocks.22.mlp.mlp.2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 8,
+        "offset": 42645504,
+        "bytes": 2048
+      }
+    },
+    "blocks.22.mlp.mlp.2.weight": {
+      "shape": [
+        1024,
+        4096
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 4096,
+      "padded_rows": 1024,
+      "padded_columns": 4096,
+      "base_data": {
+        "shard": 8,
+        "offset": 42647552,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 8,
+        "offset": 46841856,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 8,
+        "offset": 46972928,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 8,
+        "offset": 49070080,
+        "bytes": 131072
+      }
+    },
+    "blocks.22.norm2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 8,
+        "offset": 49201152,
+        "bytes": 2048
+      }
+    },
+    "blocks.22.norm2.weight": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 8,
+        "offset": 49203200,
+        "bytes": 2048
+      }
+    },
+    "blocks.22.self_attn.k_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 8,
+        "offset": 49205248,
+        "bytes": 2048
+      }
+    },
+    "blocks.22.self_attn.q_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 8,
+        "offset": 49207296,
+        "bytes": 2048
+      }
+    },
+    "blocks.22.self_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 8,
+        "offset": 49209344,
+        "bytes": 2048
+      }
+    },
+    "blocks.22.self_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 8,
+        "offset": 49211392,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 8,
+        "offset": 50259968,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 8,
+        "offset": 50292736,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 8,
+        "offset": 50817024,
+        "bytes": 32768
+      }
+    },
+    "blocks.22.self_attn.to_qkv.bias": {
+      "shape": [
+        3072
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 8,
+        "offset": 50849792,
+        "bytes": 6144
+      }
+    },
+    "blocks.22.self_attn.to_qkv.weight": {
+      "shape": [
+        3072,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 3072,
+      "columns": 1024,
+      "padded_rows": 3072,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 8,
+        "offset": 50855936,
+        "bytes": 3145728
+      },
+      "base_scales": {
+        "shard": 8,
+        "offset": 54001664,
+        "bytes": 98304
+      },
+      "residual_data": {
+        "shard": 8,
+        "offset": 54099968,
+        "bytes": 1572864
+      },
+      "residual_scales": {
+        "shard": 8,
+        "offset": 55672832,
+        "bytes": 98304
+      }
+    },
+    "blocks.23.adaLN_modulation.1.bias": {
+      "shape": [
+        6144
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 8,
+        "offset": 55771136,
+        "bytes": 12288
+      }
+    },
+    "blocks.23.adaLN_modulation.1.weight": {
+      "shape": [
+        6144,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 6144,
+      "columns": 1024,
+      "padded_rows": 6144,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 8,
+        "offset": 55783424,
+        "bytes": 6291456
+      },
+      "base_scales": {
+        "shard": 8,
+        "offset": 62074880,
+        "bytes": 196608
+      },
+      "residual_data": {
+        "shard": 8,
+        "offset": 62271488,
+        "bytes": 3145728
+      },
+      "residual_scales": {
+        "shard": 8,
+        "offset": 65417216,
+        "bytes": 196608
+      }
+    },
+    "blocks.23.cross_attn.to_kv.bias": {
+      "shape": [
+        2048
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 8,
+        "offset": 65613824,
+        "bytes": 4096
+      }
+    },
+    "blocks.23.cross_attn.to_kv.weight": {
+      "shape": [
+        2048,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 2048,
+      "columns": 1024,
+      "padded_rows": 2048,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 9,
+        "offset": 0,
+        "bytes": 2097152
+      },
+      "base_scales": {
+        "shard": 9,
+        "offset": 2097152,
+        "bytes": 65536
+      },
+      "residual_data": {
+        "shard": 9,
+        "offset": 2162688,
+        "bytes": 1048576
+      },
+      "residual_scales": {
+        "shard": 9,
+        "offset": 3211264,
+        "bytes": 65536
+      }
+    },
+    "blocks.23.cross_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 9,
+        "offset": 3276800,
+        "bytes": 2048
+      }
+    },
+    "blocks.23.cross_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 9,
+        "offset": 3278848,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 9,
+        "offset": 4327424,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 9,
+        "offset": 4360192,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 9,
+        "offset": 4884480,
+        "bytes": 32768
+      }
+    },
+    "blocks.23.cross_attn.to_q.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 9,
+        "offset": 4917248,
+        "bytes": 2048
+      }
+    },
+    "blocks.23.cross_attn.to_q.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 9,
+        "offset": 4919296,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 9,
+        "offset": 5967872,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 9,
+        "offset": 6000640,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 9,
+        "offset": 6524928,
+        "bytes": 32768
+      }
+    },
+    "blocks.23.mlp.mlp.0.bias": {
+      "shape": [
+        4096
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 9,
+        "offset": 6557696,
+        "bytes": 8192
+      }
+    },
+    "blocks.23.mlp.mlp.0.weight": {
+      "shape": [
+        4096,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 4096,
+      "columns": 1024,
+      "padded_rows": 4096,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 9,
+        "offset": 6565888,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 9,
+        "offset": 10760192,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 9,
+        "offset": 10891264,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 9,
+        "offset": 12988416,
+        "bytes": 131072
+      }
+    },
+    "blocks.23.mlp.mlp.2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 9,
+        "offset": 13119488,
+        "bytes": 2048
+      }
+    },
+    "blocks.23.mlp.mlp.2.weight": {
+      "shape": [
+        1024,
+        4096
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 4096,
+      "padded_rows": 1024,
+      "padded_columns": 4096,
+      "base_data": {
+        "shard": 9,
+        "offset": 13121536,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 9,
+        "offset": 17315840,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 9,
+        "offset": 17446912,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 9,
+        "offset": 19544064,
+        "bytes": 131072
+      }
+    },
+    "blocks.23.norm2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 9,
+        "offset": 19675136,
+        "bytes": 2048
+      }
+    },
+    "blocks.23.norm2.weight": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 9,
+        "offset": 19677184,
+        "bytes": 2048
+      }
+    },
+    "blocks.23.self_attn.k_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 9,
+        "offset": 19679232,
+        "bytes": 2048
+      }
+    },
+    "blocks.23.self_attn.q_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 9,
+        "offset": 19681280,
+        "bytes": 2048
+      }
+    },
+    "blocks.23.self_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 9,
+        "offset": 19683328,
+        "bytes": 2048
+      }
+    },
+    "blocks.23.self_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 9,
+        "offset": 19685376,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 9,
+        "offset": 20733952,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 9,
+        "offset": 20766720,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 9,
+        "offset": 21291008,
+        "bytes": 32768
+      }
+    },
+    "blocks.23.self_attn.to_qkv.bias": {
+      "shape": [
+        3072
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 9,
+        "offset": 21323776,
+        "bytes": 6144
+      }
+    },
+    "blocks.23.self_attn.to_qkv.weight": {
+      "shape": [
+        3072,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 3072,
+      "columns": 1024,
+      "padded_rows": 3072,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 9,
+        "offset": 21329920,
+        "bytes": 3145728
+      },
+      "base_scales": {
+        "shard": 9,
+        "offset": 24475648,
+        "bytes": 98304
+      },
+      "residual_data": {
+        "shard": 9,
+        "offset": 24573952,
+        "bytes": 1572864
+      },
+      "residual_scales": {
+        "shard": 9,
+        "offset": 26146816,
+        "bytes": 98304
+      }
+    },
+    "blocks.3.adaLN_modulation.1.bias": {
+      "shape": [
+        6144
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 9,
+        "offset": 26245120,
+        "bytes": 12288
+      }
+    },
+    "blocks.3.adaLN_modulation.1.weight": {
+      "shape": [
+        6144,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 6144,
+      "columns": 1024,
+      "padded_rows": 6144,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 9,
+        "offset": 26257408,
+        "bytes": 6291456
+      },
+      "base_scales": {
+        "shard": 9,
+        "offset": 32548864,
+        "bytes": 196608
+      },
+      "residual_data": {
+        "shard": 9,
+        "offset": 32745472,
+        "bytes": 3145728
+      },
+      "residual_scales": {
+        "shard": 9,
+        "offset": 35891200,
+        "bytes": 196608
+      }
+    },
+    "blocks.3.cross_attn.to_kv.bias": {
+      "shape": [
+        2048
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 9,
+        "offset": 36087808,
+        "bytes": 4096
+      }
+    },
+    "blocks.3.cross_attn.to_kv.weight": {
+      "shape": [
+        2048,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 2048,
+      "columns": 1024,
+      "padded_rows": 2048,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 9,
+        "offset": 36091904,
+        "bytes": 2097152
+      },
+      "base_scales": {
+        "shard": 9,
+        "offset": 38189056,
+        "bytes": 65536
+      },
+      "residual_data": {
+        "shard": 9,
+        "offset": 38254592,
+        "bytes": 1048576
+      },
+      "residual_scales": {
+        "shard": 9,
+        "offset": 39303168,
+        "bytes": 65536
+      }
+    },
+    "blocks.3.cross_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 9,
+        "offset": 39368704,
+        "bytes": 2048
+      }
+    },
+    "blocks.3.cross_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 9,
+        "offset": 39370752,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 9,
+        "offset": 40419328,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 9,
+        "offset": 40452096,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 9,
+        "offset": 40976384,
+        "bytes": 32768
+      }
+    },
+    "blocks.3.cross_attn.to_q.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 9,
+        "offset": 41009152,
+        "bytes": 2048
+      }
+    },
+    "blocks.3.cross_attn.to_q.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 9,
+        "offset": 41011200,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 9,
+        "offset": 42059776,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 9,
+        "offset": 42092544,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 9,
+        "offset": 42616832,
+        "bytes": 32768
+      }
+    },
+    "blocks.3.mlp.mlp.0.bias": {
+      "shape": [
+        4096
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 9,
+        "offset": 42649600,
+        "bytes": 8192
+      }
+    },
+    "blocks.3.mlp.mlp.0.weight": {
+      "shape": [
+        4096,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 4096,
+      "columns": 1024,
+      "padded_rows": 4096,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 9,
+        "offset": 42657792,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 9,
+        "offset": 46852096,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 9,
+        "offset": 46983168,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 9,
+        "offset": 49080320,
+        "bytes": 131072
+      }
+    },
+    "blocks.3.mlp.mlp.2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 9,
+        "offset": 49211392,
+        "bytes": 2048
+      }
+    },
+    "blocks.3.mlp.mlp.2.weight": {
+      "shape": [
+        1024,
+        4096
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 4096,
+      "padded_rows": 1024,
+      "padded_columns": 4096,
+      "base_data": {
+        "shard": 9,
+        "offset": 49213440,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 9,
+        "offset": 53407744,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 9,
+        "offset": 53538816,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 9,
+        "offset": 55635968,
+        "bytes": 131072
+      }
+    },
+    "blocks.3.norm2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 9,
+        "offset": 55767040,
+        "bytes": 2048
+      }
+    },
+    "blocks.3.norm2.weight": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 9,
+        "offset": 55769088,
+        "bytes": 2048
+      }
+    },
+    "blocks.3.self_attn.k_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 9,
+        "offset": 55771136,
+        "bytes": 2048
+      }
+    },
+    "blocks.3.self_attn.q_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 9,
+        "offset": 55773184,
+        "bytes": 2048
+      }
+    },
+    "blocks.3.self_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 9,
+        "offset": 55775232,
+        "bytes": 2048
+      }
+    },
+    "blocks.3.self_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 9,
+        "offset": 55777280,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 9,
+        "offset": 56825856,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 9,
+        "offset": 56858624,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 9,
+        "offset": 57382912,
+        "bytes": 32768
+      }
+    },
+    "blocks.3.self_attn.to_qkv.bias": {
+      "shape": [
+        3072
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 9,
+        "offset": 57415680,
+        "bytes": 6144
+      }
+    },
+    "blocks.3.self_attn.to_qkv.weight": {
+      "shape": [
+        3072,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 3072,
+      "columns": 1024,
+      "padded_rows": 3072,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 9,
+        "offset": 57421824,
+        "bytes": 3145728
+      },
+      "base_scales": {
+        "shard": 9,
+        "offset": 60567552,
+        "bytes": 98304
+      },
+      "residual_data": {
+        "shard": 9,
+        "offset": 60665856,
+        "bytes": 1572864
+      },
+      "residual_scales": {
+        "shard": 9,
+        "offset": 62238720,
+        "bytes": 98304
+      }
+    },
+    "blocks.4.adaLN_modulation.1.bias": {
+      "shape": [
+        6144
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 9,
+        "offset": 62337024,
+        "bytes": 12288
+      }
+    },
+    "blocks.4.adaLN_modulation.1.weight": {
+      "shape": [
+        6144,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 6144,
+      "columns": 1024,
+      "padded_rows": 6144,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 10,
+        "offset": 0,
+        "bytes": 6291456
+      },
+      "base_scales": {
+        "shard": 10,
+        "offset": 6291456,
+        "bytes": 196608
+      },
+      "residual_data": {
+        "shard": 10,
+        "offset": 6488064,
+        "bytes": 3145728
+      },
+      "residual_scales": {
+        "shard": 10,
+        "offset": 9633792,
+        "bytes": 196608
+      }
+    },
+    "blocks.4.cross_attn.to_kv.bias": {
+      "shape": [
+        2048
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 10,
+        "offset": 9830400,
+        "bytes": 4096
+      }
+    },
+    "blocks.4.cross_attn.to_kv.weight": {
+      "shape": [
+        2048,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 2048,
+      "columns": 1024,
+      "padded_rows": 2048,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 10,
+        "offset": 9834496,
+        "bytes": 2097152
+      },
+      "base_scales": {
+        "shard": 10,
+        "offset": 11931648,
+        "bytes": 65536
+      },
+      "residual_data": {
+        "shard": 10,
+        "offset": 11997184,
+        "bytes": 1048576
+      },
+      "residual_scales": {
+        "shard": 10,
+        "offset": 13045760,
+        "bytes": 65536
+      }
+    },
+    "blocks.4.cross_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 10,
+        "offset": 13111296,
+        "bytes": 2048
+      }
+    },
+    "blocks.4.cross_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 10,
+        "offset": 13113344,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 10,
+        "offset": 14161920,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 10,
+        "offset": 14194688,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 10,
+        "offset": 14718976,
+        "bytes": 32768
+      }
+    },
+    "blocks.4.cross_attn.to_q.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 10,
+        "offset": 14751744,
+        "bytes": 2048
+      }
+    },
+    "blocks.4.cross_attn.to_q.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 10,
+        "offset": 14753792,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 10,
+        "offset": 15802368,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 10,
+        "offset": 15835136,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 10,
+        "offset": 16359424,
+        "bytes": 32768
+      }
+    },
+    "blocks.4.mlp.mlp.0.bias": {
+      "shape": [
+        4096
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 10,
+        "offset": 16392192,
+        "bytes": 8192
+      }
+    },
+    "blocks.4.mlp.mlp.0.weight": {
+      "shape": [
+        4096,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 4096,
+      "columns": 1024,
+      "padded_rows": 4096,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 10,
+        "offset": 16400384,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 10,
+        "offset": 20594688,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 10,
+        "offset": 20725760,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 10,
+        "offset": 22822912,
+        "bytes": 131072
+      }
+    },
+    "blocks.4.mlp.mlp.2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 10,
+        "offset": 22953984,
+        "bytes": 2048
+      }
+    },
+    "blocks.4.mlp.mlp.2.weight": {
+      "shape": [
+        1024,
+        4096
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 4096,
+      "padded_rows": 1024,
+      "padded_columns": 4096,
+      "base_data": {
+        "shard": 10,
+        "offset": 22956032,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 10,
+        "offset": 27150336,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 10,
+        "offset": 27281408,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 10,
+        "offset": 29378560,
+        "bytes": 131072
+      }
+    },
+    "blocks.4.norm2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 10,
+        "offset": 29509632,
+        "bytes": 2048
+      }
+    },
+    "blocks.4.norm2.weight": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 10,
+        "offset": 29511680,
+        "bytes": 2048
+      }
+    },
+    "blocks.4.self_attn.k_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 10,
+        "offset": 29513728,
+        "bytes": 2048
+      }
+    },
+    "blocks.4.self_attn.q_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 10,
+        "offset": 29515776,
+        "bytes": 2048
+      }
+    },
+    "blocks.4.self_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 10,
+        "offset": 29517824,
+        "bytes": 2048
+      }
+    },
+    "blocks.4.self_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 10,
+        "offset": 29519872,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 10,
+        "offset": 30568448,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 10,
+        "offset": 30601216,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 10,
+        "offset": 31125504,
+        "bytes": 32768
+      }
+    },
+    "blocks.4.self_attn.to_qkv.bias": {
+      "shape": [
+        3072
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 10,
+        "offset": 31158272,
+        "bytes": 6144
+      }
+    },
+    "blocks.4.self_attn.to_qkv.weight": {
+      "shape": [
+        3072,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 3072,
+      "columns": 1024,
+      "padded_rows": 3072,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 10,
+        "offset": 31164416,
+        "bytes": 3145728
+      },
+      "base_scales": {
+        "shard": 10,
+        "offset": 34310144,
+        "bytes": 98304
+      },
+      "residual_data": {
+        "shard": 10,
+        "offset": 34408448,
+        "bytes": 1572864
+      },
+      "residual_scales": {
+        "shard": 10,
+        "offset": 35981312,
+        "bytes": 98304
+      }
+    },
+    "blocks.5.adaLN_modulation.1.bias": {
+      "shape": [
+        6144
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 10,
+        "offset": 36079616,
+        "bytes": 12288
+      }
+    },
+    "blocks.5.adaLN_modulation.1.weight": {
+      "shape": [
+        6144,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 6144,
+      "columns": 1024,
+      "padded_rows": 6144,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 10,
+        "offset": 36091904,
+        "bytes": 6291456
+      },
+      "base_scales": {
+        "shard": 10,
+        "offset": 42383360,
+        "bytes": 196608
+      },
+      "residual_data": {
+        "shard": 10,
+        "offset": 42579968,
+        "bytes": 3145728
+      },
+      "residual_scales": {
+        "shard": 10,
+        "offset": 45725696,
+        "bytes": 196608
+      }
+    },
+    "blocks.5.cross_attn.to_kv.bias": {
+      "shape": [
+        2048
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 10,
+        "offset": 45922304,
+        "bytes": 4096
+      }
+    },
+    "blocks.5.cross_attn.to_kv.weight": {
+      "shape": [
+        2048,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 2048,
+      "columns": 1024,
+      "padded_rows": 2048,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 10,
+        "offset": 45926400,
+        "bytes": 2097152
+      },
+      "base_scales": {
+        "shard": 10,
+        "offset": 48023552,
+        "bytes": 65536
+      },
+      "residual_data": {
+        "shard": 10,
+        "offset": 48089088,
+        "bytes": 1048576
+      },
+      "residual_scales": {
+        "shard": 10,
+        "offset": 49137664,
+        "bytes": 65536
+      }
+    },
+    "blocks.5.cross_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 10,
+        "offset": 49203200,
+        "bytes": 2048
+      }
+    },
+    "blocks.5.cross_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 10,
+        "offset": 49205248,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 10,
+        "offset": 50253824,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 10,
+        "offset": 50286592,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 10,
+        "offset": 50810880,
+        "bytes": 32768
+      }
+    },
+    "blocks.5.cross_attn.to_q.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 10,
+        "offset": 50843648,
+        "bytes": 2048
+      }
+    },
+    "blocks.5.cross_attn.to_q.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 10,
+        "offset": 50845696,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 10,
+        "offset": 51894272,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 10,
+        "offset": 51927040,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 10,
+        "offset": 52451328,
+        "bytes": 32768
+      }
+    },
+    "blocks.5.mlp.mlp.0.bias": {
+      "shape": [
+        4096
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 10,
+        "offset": 52484096,
+        "bytes": 8192
+      }
+    },
+    "blocks.5.mlp.mlp.0.weight": {
+      "shape": [
+        4096,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 4096,
+      "columns": 1024,
+      "padded_rows": 4096,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 10,
+        "offset": 52492288,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 10,
+        "offset": 56686592,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 10,
+        "offset": 56817664,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 10,
+        "offset": 58914816,
+        "bytes": 131072
+      }
+    },
+    "blocks.5.mlp.mlp.2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 10,
+        "offset": 59045888,
+        "bytes": 2048
+      }
+    },
+    "blocks.5.mlp.mlp.2.weight": {
+      "shape": [
+        1024,
+        4096
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 4096,
+      "padded_rows": 1024,
+      "padded_columns": 4096,
+      "base_data": {
+        "shard": 10,
+        "offset": 59047936,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 10,
+        "offset": 63242240,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 10,
+        "offset": 63373312,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 10,
+        "offset": 65470464,
+        "bytes": 131072
+      }
+    },
+    "blocks.5.norm2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 10,
+        "offset": 65601536,
+        "bytes": 2048
+      }
+    },
+    "blocks.5.norm2.weight": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 10,
+        "offset": 65603584,
+        "bytes": 2048
+      }
+    },
+    "blocks.5.self_attn.k_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 10,
+        "offset": 65605632,
+        "bytes": 2048
+      }
+    },
+    "blocks.5.self_attn.q_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 10,
+        "offset": 65607680,
+        "bytes": 2048
+      }
+    },
+    "blocks.5.self_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 10,
+        "offset": 65609728,
+        "bytes": 2048
+      }
+    },
+    "blocks.5.self_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 10,
+        "offset": 65611776,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 10,
+        "offset": 66660352,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 11,
+        "offset": 0,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 11,
+        "offset": 524288,
+        "bytes": 32768
+      }
+    },
+    "blocks.5.self_attn.to_qkv.bias": {
+      "shape": [
+        3072
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 11,
+        "offset": 557056,
+        "bytes": 6144
+      }
+    },
+    "blocks.5.self_attn.to_qkv.weight": {
+      "shape": [
+        3072,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 3072,
+      "columns": 1024,
+      "padded_rows": 3072,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 11,
+        "offset": 563200,
+        "bytes": 3145728
+      },
+      "base_scales": {
+        "shard": 11,
+        "offset": 3708928,
+        "bytes": 98304
+      },
+      "residual_data": {
+        "shard": 11,
+        "offset": 3807232,
+        "bytes": 1572864
+      },
+      "residual_scales": {
+        "shard": 11,
+        "offset": 5380096,
+        "bytes": 98304
+      }
+    },
+    "blocks.6.adaLN_modulation.1.bias": {
+      "shape": [
+        6144
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 11,
+        "offset": 5478400,
+        "bytes": 12288
+      }
+    },
+    "blocks.6.adaLN_modulation.1.weight": {
+      "shape": [
+        6144,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 6144,
+      "columns": 1024,
+      "padded_rows": 6144,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 11,
+        "offset": 5490688,
+        "bytes": 6291456
+      },
+      "base_scales": {
+        "shard": 11,
+        "offset": 11782144,
+        "bytes": 196608
+      },
+      "residual_data": {
+        "shard": 11,
+        "offset": 11978752,
+        "bytes": 3145728
+      },
+      "residual_scales": {
+        "shard": 11,
+        "offset": 15124480,
+        "bytes": 196608
+      }
+    },
+    "blocks.6.cross_attn.to_kv.bias": {
+      "shape": [
+        2048
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 11,
+        "offset": 15321088,
+        "bytes": 4096
+      }
+    },
+    "blocks.6.cross_attn.to_kv.weight": {
+      "shape": [
+        2048,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 2048,
+      "columns": 1024,
+      "padded_rows": 2048,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 11,
+        "offset": 15325184,
+        "bytes": 2097152
+      },
+      "base_scales": {
+        "shard": 11,
+        "offset": 17422336,
+        "bytes": 65536
+      },
+      "residual_data": {
+        "shard": 11,
+        "offset": 17487872,
+        "bytes": 1048576
+      },
+      "residual_scales": {
+        "shard": 11,
+        "offset": 18536448,
+        "bytes": 65536
+      }
+    },
+    "blocks.6.cross_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 11,
+        "offset": 18601984,
+        "bytes": 2048
+      }
+    },
+    "blocks.6.cross_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 11,
+        "offset": 18604032,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 11,
+        "offset": 19652608,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 11,
+        "offset": 19685376,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 11,
+        "offset": 20209664,
+        "bytes": 32768
+      }
+    },
+    "blocks.6.cross_attn.to_q.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 11,
+        "offset": 20242432,
+        "bytes": 2048
+      }
+    },
+    "blocks.6.cross_attn.to_q.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 11,
+        "offset": 20244480,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 11,
+        "offset": 21293056,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 11,
+        "offset": 21325824,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 11,
+        "offset": 21850112,
+        "bytes": 32768
+      }
+    },
+    "blocks.6.mlp.mlp.0.bias": {
+      "shape": [
+        4096
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 11,
+        "offset": 21882880,
+        "bytes": 8192
+      }
+    },
+    "blocks.6.mlp.mlp.0.weight": {
+      "shape": [
+        4096,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 4096,
+      "columns": 1024,
+      "padded_rows": 4096,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 11,
+        "offset": 21891072,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 11,
+        "offset": 26085376,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 11,
+        "offset": 26216448,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 11,
+        "offset": 28313600,
+        "bytes": 131072
+      }
+    },
+    "blocks.6.mlp.mlp.2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 11,
+        "offset": 28444672,
+        "bytes": 2048
+      }
+    },
+    "blocks.6.mlp.mlp.2.weight": {
+      "shape": [
+        1024,
+        4096
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 4096,
+      "padded_rows": 1024,
+      "padded_columns": 4096,
+      "base_data": {
+        "shard": 11,
+        "offset": 28446720,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 11,
+        "offset": 32641024,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 11,
+        "offset": 32772096,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 11,
+        "offset": 34869248,
+        "bytes": 131072
+      }
+    },
+    "blocks.6.norm2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 11,
+        "offset": 35000320,
+        "bytes": 2048
+      }
+    },
+    "blocks.6.norm2.weight": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 11,
+        "offset": 35002368,
+        "bytes": 2048
+      }
+    },
+    "blocks.6.self_attn.k_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 11,
+        "offset": 35004416,
+        "bytes": 2048
+      }
+    },
+    "blocks.6.self_attn.q_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 11,
+        "offset": 35006464,
+        "bytes": 2048
+      }
+    },
+    "blocks.6.self_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 11,
+        "offset": 35008512,
+        "bytes": 2048
+      }
+    },
+    "blocks.6.self_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 11,
+        "offset": 35010560,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 11,
+        "offset": 36059136,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 11,
+        "offset": 36091904,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 11,
+        "offset": 36616192,
+        "bytes": 32768
+      }
+    },
+    "blocks.6.self_attn.to_qkv.bias": {
+      "shape": [
+        3072
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 11,
+        "offset": 36648960,
+        "bytes": 6144
+      }
+    },
+    "blocks.6.self_attn.to_qkv.weight": {
+      "shape": [
+        3072,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 3072,
+      "columns": 1024,
+      "padded_rows": 3072,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 11,
+        "offset": 36655104,
+        "bytes": 3145728
+      },
+      "base_scales": {
+        "shard": 11,
+        "offset": 39800832,
+        "bytes": 98304
+      },
+      "residual_data": {
+        "shard": 11,
+        "offset": 39899136,
+        "bytes": 1572864
+      },
+      "residual_scales": {
+        "shard": 11,
+        "offset": 41472000,
+        "bytes": 98304
+      }
+    },
+    "blocks.7.adaLN_modulation.1.bias": {
+      "shape": [
+        6144
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 11,
+        "offset": 41570304,
+        "bytes": 12288
+      }
+    },
+    "blocks.7.adaLN_modulation.1.weight": {
+      "shape": [
+        6144,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 6144,
+      "columns": 1024,
+      "padded_rows": 6144,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 11,
+        "offset": 41582592,
+        "bytes": 6291456
+      },
+      "base_scales": {
+        "shard": 11,
+        "offset": 47874048,
+        "bytes": 196608
+      },
+      "residual_data": {
+        "shard": 11,
+        "offset": 48070656,
+        "bytes": 3145728
+      },
+      "residual_scales": {
+        "shard": 11,
+        "offset": 51216384,
+        "bytes": 196608
+      }
+    },
+    "blocks.7.cross_attn.to_kv.bias": {
+      "shape": [
+        2048
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 11,
+        "offset": 51412992,
+        "bytes": 4096
+      }
+    },
+    "blocks.7.cross_attn.to_kv.weight": {
+      "shape": [
+        2048,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 2048,
+      "columns": 1024,
+      "padded_rows": 2048,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 11,
+        "offset": 51417088,
+        "bytes": 2097152
+      },
+      "base_scales": {
+        "shard": 11,
+        "offset": 53514240,
+        "bytes": 65536
+      },
+      "residual_data": {
+        "shard": 11,
+        "offset": 53579776,
+        "bytes": 1048576
+      },
+      "residual_scales": {
+        "shard": 11,
+        "offset": 54628352,
+        "bytes": 65536
+      }
+    },
+    "blocks.7.cross_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 11,
+        "offset": 54693888,
+        "bytes": 2048
+      }
+    },
+    "blocks.7.cross_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 11,
+        "offset": 54695936,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 11,
+        "offset": 55744512,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 11,
+        "offset": 55777280,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 11,
+        "offset": 56301568,
+        "bytes": 32768
+      }
+    },
+    "blocks.7.cross_attn.to_q.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 11,
+        "offset": 56334336,
+        "bytes": 2048
+      }
+    },
+    "blocks.7.cross_attn.to_q.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 11,
+        "offset": 56336384,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 11,
+        "offset": 57384960,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 11,
+        "offset": 57417728,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 11,
+        "offset": 57942016,
+        "bytes": 32768
+      }
+    },
+    "blocks.7.mlp.mlp.0.bias": {
+      "shape": [
+        4096
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 11,
+        "offset": 57974784,
+        "bytes": 8192
+      }
+    },
+    "blocks.7.mlp.mlp.0.weight": {
+      "shape": [
+        4096,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 4096,
+      "columns": 1024,
+      "padded_rows": 4096,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 11,
+        "offset": 57982976,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 11,
+        "offset": 62177280,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 11,
+        "offset": 62308352,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 11,
+        "offset": 64405504,
+        "bytes": 131072
+      }
+    },
+    "blocks.7.mlp.mlp.2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 11,
+        "offset": 64536576,
+        "bytes": 2048
+      }
+    },
+    "blocks.7.mlp.mlp.2.weight": {
+      "shape": [
+        1024,
+        4096
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 4096,
+      "padded_rows": 1024,
+      "padded_columns": 4096,
+      "base_data": {
+        "shard": 12,
+        "offset": 0,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 12,
+        "offset": 4194304,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 12,
+        "offset": 4325376,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 12,
+        "offset": 6422528,
+        "bytes": 131072
+      }
+    },
+    "blocks.7.norm2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 12,
+        "offset": 6553600,
+        "bytes": 2048
+      }
+    },
+    "blocks.7.norm2.weight": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 12,
+        "offset": 6555648,
+        "bytes": 2048
+      }
+    },
+    "blocks.7.self_attn.k_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 12,
+        "offset": 6557696,
+        "bytes": 2048
+      }
+    },
+    "blocks.7.self_attn.q_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 12,
+        "offset": 6559744,
+        "bytes": 2048
+      }
+    },
+    "blocks.7.self_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 12,
+        "offset": 6561792,
+        "bytes": 2048
+      }
+    },
+    "blocks.7.self_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 12,
+        "offset": 6563840,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 12,
+        "offset": 7612416,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 12,
+        "offset": 7645184,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 12,
+        "offset": 8169472,
+        "bytes": 32768
+      }
+    },
+    "blocks.7.self_attn.to_qkv.bias": {
+      "shape": [
+        3072
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 12,
+        "offset": 8202240,
+        "bytes": 6144
+      }
+    },
+    "blocks.7.self_attn.to_qkv.weight": {
+      "shape": [
+        3072,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 3072,
+      "columns": 1024,
+      "padded_rows": 3072,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 12,
+        "offset": 8208384,
+        "bytes": 3145728
+      },
+      "base_scales": {
+        "shard": 12,
+        "offset": 11354112,
+        "bytes": 98304
+      },
+      "residual_data": {
+        "shard": 12,
+        "offset": 11452416,
+        "bytes": 1572864
+      },
+      "residual_scales": {
+        "shard": 12,
+        "offset": 13025280,
+        "bytes": 98304
+      }
+    },
+    "blocks.8.adaLN_modulation.1.bias": {
+      "shape": [
+        6144
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 12,
+        "offset": 13123584,
+        "bytes": 12288
+      }
+    },
+    "blocks.8.adaLN_modulation.1.weight": {
+      "shape": [
+        6144,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 6144,
+      "columns": 1024,
+      "padded_rows": 6144,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 12,
+        "offset": 13135872,
+        "bytes": 6291456
+      },
+      "base_scales": {
+        "shard": 12,
+        "offset": 19427328,
+        "bytes": 196608
+      },
+      "residual_data": {
+        "shard": 12,
+        "offset": 19623936,
+        "bytes": 3145728
+      },
+      "residual_scales": {
+        "shard": 12,
+        "offset": 22769664,
+        "bytes": 196608
+      }
+    },
+    "blocks.8.cross_attn.to_kv.bias": {
+      "shape": [
+        2048
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 12,
+        "offset": 22966272,
+        "bytes": 4096
+      }
+    },
+    "blocks.8.cross_attn.to_kv.weight": {
+      "shape": [
+        2048,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 2048,
+      "columns": 1024,
+      "padded_rows": 2048,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 12,
+        "offset": 22970368,
+        "bytes": 2097152
+      },
+      "base_scales": {
+        "shard": 12,
+        "offset": 25067520,
+        "bytes": 65536
+      },
+      "residual_data": {
+        "shard": 12,
+        "offset": 25133056,
+        "bytes": 1048576
+      },
+      "residual_scales": {
+        "shard": 12,
+        "offset": 26181632,
+        "bytes": 65536
+      }
+    },
+    "blocks.8.cross_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 12,
+        "offset": 26247168,
+        "bytes": 2048
+      }
+    },
+    "blocks.8.cross_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 12,
+        "offset": 26249216,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 12,
+        "offset": 27297792,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 12,
+        "offset": 27330560,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 12,
+        "offset": 27854848,
+        "bytes": 32768
+      }
+    },
+    "blocks.8.cross_attn.to_q.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 12,
+        "offset": 27887616,
+        "bytes": 2048
+      }
+    },
+    "blocks.8.cross_attn.to_q.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 12,
+        "offset": 27889664,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 12,
+        "offset": 28938240,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 12,
+        "offset": 28971008,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 12,
+        "offset": 29495296,
+        "bytes": 32768
+      }
+    },
+    "blocks.8.mlp.mlp.0.bias": {
+      "shape": [
+        4096
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 12,
+        "offset": 29528064,
+        "bytes": 8192
+      }
+    },
+    "blocks.8.mlp.mlp.0.weight": {
+      "shape": [
+        4096,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 4096,
+      "columns": 1024,
+      "padded_rows": 4096,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 12,
+        "offset": 29536256,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 12,
+        "offset": 33730560,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 12,
+        "offset": 33861632,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 12,
+        "offset": 35958784,
+        "bytes": 131072
+      }
+    },
+    "blocks.8.mlp.mlp.2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 12,
+        "offset": 36089856,
+        "bytes": 2048
+      }
+    },
+    "blocks.8.mlp.mlp.2.weight": {
+      "shape": [
+        1024,
+        4096
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 4096,
+      "padded_rows": 1024,
+      "padded_columns": 4096,
+      "base_data": {
+        "shard": 12,
+        "offset": 36091904,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 12,
+        "offset": 40286208,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 12,
+        "offset": 40417280,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 12,
+        "offset": 42514432,
+        "bytes": 131072
+      }
+    },
+    "blocks.8.norm2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 12,
+        "offset": 42645504,
+        "bytes": 2048
+      }
+    },
+    "blocks.8.norm2.weight": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 12,
+        "offset": 42647552,
+        "bytes": 2048
+      }
+    },
+    "blocks.8.self_attn.k_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 12,
+        "offset": 42649600,
+        "bytes": 2048
+      }
+    },
+    "blocks.8.self_attn.q_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 12,
+        "offset": 42651648,
+        "bytes": 2048
+      }
+    },
+    "blocks.8.self_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 12,
+        "offset": 42653696,
+        "bytes": 2048
+      }
+    },
+    "blocks.8.self_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 12,
+        "offset": 42655744,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 12,
+        "offset": 43704320,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 12,
+        "offset": 43737088,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 12,
+        "offset": 44261376,
+        "bytes": 32768
+      }
+    },
+    "blocks.8.self_attn.to_qkv.bias": {
+      "shape": [
+        3072
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 12,
+        "offset": 44294144,
+        "bytes": 6144
+      }
+    },
+    "blocks.8.self_attn.to_qkv.weight": {
+      "shape": [
+        3072,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 3072,
+      "columns": 1024,
+      "padded_rows": 3072,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 12,
+        "offset": 44300288,
+        "bytes": 3145728
+      },
+      "base_scales": {
+        "shard": 12,
+        "offset": 47446016,
+        "bytes": 98304
+      },
+      "residual_data": {
+        "shard": 12,
+        "offset": 47544320,
+        "bytes": 1572864
+      },
+      "residual_scales": {
+        "shard": 12,
+        "offset": 49117184,
+        "bytes": 98304
+      }
+    },
+    "blocks.9.adaLN_modulation.1.bias": {
+      "shape": [
+        6144
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 12,
+        "offset": 49215488,
+        "bytes": 12288
+      }
+    },
+    "blocks.9.adaLN_modulation.1.weight": {
+      "shape": [
+        6144,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 6144,
+      "columns": 1024,
+      "padded_rows": 6144,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 12,
+        "offset": 49227776,
+        "bytes": 6291456
+      },
+      "base_scales": {
+        "shard": 12,
+        "offset": 55519232,
+        "bytes": 196608
+      },
+      "residual_data": {
+        "shard": 12,
+        "offset": 55715840,
+        "bytes": 3145728
+      },
+      "residual_scales": {
+        "shard": 12,
+        "offset": 58861568,
+        "bytes": 196608
+      }
+    },
+    "blocks.9.cross_attn.to_kv.bias": {
+      "shape": [
+        2048
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 12,
+        "offset": 59058176,
+        "bytes": 4096
+      }
+    },
+    "blocks.9.cross_attn.to_kv.weight": {
+      "shape": [
+        2048,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 2048,
+      "columns": 1024,
+      "padded_rows": 2048,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 12,
+        "offset": 59062272,
+        "bytes": 2097152
+      },
+      "base_scales": {
+        "shard": 12,
+        "offset": 61159424,
+        "bytes": 65536
+      },
+      "residual_data": {
+        "shard": 12,
+        "offset": 61224960,
+        "bytes": 1048576
+      },
+      "residual_scales": {
+        "shard": 12,
+        "offset": 62273536,
+        "bytes": 65536
+      }
+    },
+    "blocks.9.cross_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 12,
+        "offset": 62339072,
+        "bytes": 2048
+      }
+    },
+    "blocks.9.cross_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 12,
+        "offset": 62341120,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 12,
+        "offset": 63389696,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 12,
+        "offset": 63422464,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 12,
+        "offset": 63946752,
+        "bytes": 32768
+      }
+    },
+    "blocks.9.cross_attn.to_q.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 12,
+        "offset": 63979520,
+        "bytes": 2048
+      }
+    },
+    "blocks.9.cross_attn.to_q.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 12,
+        "offset": 63981568,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 12,
+        "offset": 65030144,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 12,
+        "offset": 65062912,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 12,
+        "offset": 65587200,
+        "bytes": 32768
+      }
+    },
+    "blocks.9.mlp.mlp.0.bias": {
+      "shape": [
+        4096
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 12,
+        "offset": 65619968,
+        "bytes": 8192
+      }
+    },
+    "blocks.9.mlp.mlp.0.weight": {
+      "shape": [
+        4096,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 4096,
+      "columns": 1024,
+      "padded_rows": 4096,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 13,
+        "offset": 0,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 13,
+        "offset": 4194304,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 13,
+        "offset": 4325376,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 13,
+        "offset": 6422528,
+        "bytes": 131072
+      }
+    },
+    "blocks.9.mlp.mlp.2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 13,
+        "offset": 6553600,
+        "bytes": 2048
+      }
+    },
+    "blocks.9.mlp.mlp.2.weight": {
+      "shape": [
+        1024,
+        4096
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 4096,
+      "padded_rows": 1024,
+      "padded_columns": 4096,
+      "base_data": {
+        "shard": 13,
+        "offset": 6555648,
+        "bytes": 4194304
+      },
+      "base_scales": {
+        "shard": 13,
+        "offset": 10749952,
+        "bytes": 131072
+      },
+      "residual_data": {
+        "shard": 13,
+        "offset": 10881024,
+        "bytes": 2097152
+      },
+      "residual_scales": {
+        "shard": 13,
+        "offset": 12978176,
+        "bytes": 131072
+      }
+    },
+    "blocks.9.norm2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 13,
+        "offset": 13109248,
+        "bytes": 2048
+      }
+    },
+    "blocks.9.norm2.weight": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 13,
+        "offset": 13111296,
+        "bytes": 2048
+      }
+    },
+    "blocks.9.self_attn.k_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 13,
+        "offset": 13113344,
+        "bytes": 2048
+      }
+    },
+    "blocks.9.self_attn.q_rms_norm.gamma": {
+      "shape": [
+        16,
+        64
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 13,
+        "offset": 13115392,
+        "bytes": 2048
+      }
+    },
+    "blocks.9.self_attn.to_out.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 13,
+        "offset": 13117440,
+        "bytes": 2048
+      }
+    },
+    "blocks.9.self_attn.to_out.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 13,
+        "offset": 13119488,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 13,
+        "offset": 14168064,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 13,
+        "offset": 14200832,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 13,
+        "offset": 14725120,
+        "bytes": 32768
+      }
+    },
+    "blocks.9.self_attn.to_qkv.bias": {
+      "shape": [
+        3072
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 13,
+        "offset": 14757888,
+        "bytes": 6144
+      }
+    },
+    "blocks.9.self_attn.to_qkv.weight": {
+      "shape": [
+        3072,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 3072,
+      "columns": 1024,
+      "padded_rows": 3072,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 13,
+        "offset": 14764032,
+        "bytes": 3145728
+      },
+      "base_scales": {
+        "shard": 13,
+        "offset": 17909760,
+        "bytes": 98304
+      },
+      "residual_data": {
+        "shard": 13,
+        "offset": 18008064,
+        "bytes": 1572864
+      },
+      "residual_scales": {
+        "shard": 13,
+        "offset": 19580928,
+        "bytes": 98304
+      }
+    },
+    "input_layer.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 13,
+        "offset": 19679232,
+        "bytes": 2048
+      }
+    },
+    "input_layer.weight": {
+      "shape": [
+        1024,
+        8
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 8,
+      "padded_rows": 1024,
+      "padded_columns": 64,
+      "base_data": {
+        "shard": 13,
+        "offset": 19681280,
+        "bytes": 65536
+      },
+      "base_scales": {
+        "shard": 13,
+        "offset": 19746816,
+        "bytes": 2048
+      },
+      "residual_data": {
+        "shard": 13,
+        "offset": 19748864,
+        "bytes": 32768
+      },
+      "residual_scales": {
+        "shard": 13,
+        "offset": 19781632,
+        "bytes": 2048
+      }
+    },
+    "out_layer.bias": {
+      "shape": [
+        8
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 13,
+        "offset": 19783680,
+        "bytes": 16
+      }
+    },
+    "out_layer.weight": {
+      "shape": [
+        8,
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 13,
+        "offset": 19783936,
+        "bytes": 16384
+      }
+    },
+    "pos_emb": {
+      "shape": [
+        4096,
+        1024
+      ],
+      "encoding": "symmetric-quantized",
+      "bits": 8,
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 4096,
+      "columns": 1024,
+      "padded_rows": 4096,
+      "padded_columns": 1024,
+      "data": {
+        "shard": 13,
+        "offset": 19800320,
+        "bytes": 4194304
+      },
+      "scales": {
+        "shard": 13,
+        "offset": 23994624,
+        "bytes": 131072
+      }
+    },
+    "t_embedder.mlp.0.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 13,
+        "offset": 24125696,
+        "bytes": 2048
+      }
+    },
+    "t_embedder.mlp.0.weight": {
+      "shape": [
+        1024,
+        256
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 256,
+      "padded_rows": 1024,
+      "padded_columns": 256,
+      "base_data": {
+        "shard": 13,
+        "offset": 24127744,
+        "bytes": 262144
+      },
+      "base_scales": {
+        "shard": 13,
+        "offset": 24389888,
+        "bytes": 8192
+      },
+      "residual_data": {
+        "shard": 13,
+        "offset": 24398080,
+        "bytes": 131072
+      },
+      "residual_scales": {
+        "shard": 13,
+        "offset": 24529152,
+        "bytes": 8192
+      }
+    },
+    "t_embedder.mlp.2.bias": {
+      "shape": [
+        1024
+      ],
+      "encoding": "raw",
+      "dtype": "f16",
+      "data": {
+        "shard": 13,
+        "offset": 24537344,
+        "bytes": 2048
+      }
+    },
+    "t_embedder.mlp.2.weight": {
+      "shape": [
+        1024,
+        1024
+      ],
+      "encoding": "residual-quantized",
+      "group_size": 64,
+      "output_tile": 16,
+      "rows": 1024,
+      "columns": 1024,
+      "padded_rows": 1024,
+      "padded_columns": 1024,
+      "base_data": {
+        "shard": 13,
+        "offset": 24539392,
+        "bytes": 1048576
+      },
+      "base_scales": {
+        "shard": 13,
+        "offset": 25587968,
+        "bytes": 32768
+      },
+      "residual_data": {
+        "shard": 13,
+        "offset": 25620736,
+        "bytes": 524288
+      },
+      "residual_scales": {
+        "shard": 13,
+        "offset": 26145024,
+        "bytes": 32768
+      }
+    }
+  }
+}