| { |
| "algorithm": "dflash", |
| "base_model": "CohereLabs/North-Mini-Code-1.0-w4a16", |
| "checkpoint_epoch": 4, |
| "default_speculative_tokens": 3, |
| "largest_tensors": [ |
| { |
| "dtype": "BF16", |
| "name": "embed_tokens.weight", |
| "parameters": 536870912, |
| "shape": [ |
| 262144, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "lm_head.weight", |
| "parameters": 65536000, |
| "shape": [ |
| 32000, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "fc.weight", |
| "parameters": 12582912, |
| "shape": [ |
| 2048, |
| 6144 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.0.self_attn.o_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 2048, |
| 4096 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.0.self_attn.q_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 4096, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.1.self_attn.o_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 2048, |
| 4096 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.1.self_attn.q_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 4096, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.2.self_attn.o_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 2048, |
| 4096 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.2.self_attn.q_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 4096, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.3.self_attn.o_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 2048, |
| 4096 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.3.self_attn.q_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 4096, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.4.self_attn.o_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 2048, |
| 4096 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.4.self_attn.q_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 4096, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.0.mlp.down_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 2048, |
| 768 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.0.mlp.gate_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 768, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.0.mlp.up_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 768, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.1.mlp.down_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 2048, |
| 768 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.1.mlp.gate_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 768, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.1.mlp.up_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 768, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.2.mlp.down_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 2048, |
| 768 |
| ] |
| } |
| ], |
| "model_family": "North Mini Code speculative draft", |
| "model_file": { |
| "name": "model.safetensors", |
| "safetensors_metadata": { |
| "format": "pt" |
| }, |
| "sha256": "0ac5964c48003f2eab08c4691357dedc11227d88302ea7c76fc47aaddab6d5b9", |
| "size_bytes": 1466485696 |
| }, |
| "parameter_count": 733274624, |
| "parameters_by_dtype": { |
| "BF16": 732980480, |
| "BOOL": 262144, |
| "I64": 32000 |
| }, |
| "schema_version": 1, |
| "tensor_count": 62, |
| "tensors": [ |
| { |
| "dtype": "I64", |
| "name": "d2t", |
| "parameters": 32000, |
| "shape": [ |
| 32000 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "embed_tokens.weight", |
| "parameters": 536870912, |
| "shape": [ |
| 262144, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "fc.weight", |
| "parameters": 12582912, |
| "shape": [ |
| 2048, |
| 6144 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "hidden_norm.weight", |
| "parameters": 2048, |
| "shape": [ |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.0.input_layernorm.weight", |
| "parameters": 2048, |
| "shape": [ |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.0.mlp.down_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 2048, |
| 768 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.0.mlp.gate_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 768, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.0.mlp.up_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 768, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.0.post_attention_layernorm.weight", |
| "parameters": 2048, |
| "shape": [ |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.0.self_attn.k_norm.weight", |
| "parameters": 128, |
| "shape": [ |
| 128 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.0.self_attn.k_proj.weight", |
| "parameters": 1048576, |
| "shape": [ |
| 512, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.0.self_attn.o_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 2048, |
| 4096 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.0.self_attn.q_norm.weight", |
| "parameters": 128, |
| "shape": [ |
| 128 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.0.self_attn.q_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 4096, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.0.self_attn.v_proj.weight", |
| "parameters": 1048576, |
| "shape": [ |
| 512, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.1.input_layernorm.weight", |
| "parameters": 2048, |
| "shape": [ |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.1.mlp.down_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 2048, |
| 768 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.1.mlp.gate_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 768, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.1.mlp.up_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 768, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.1.post_attention_layernorm.weight", |
| "parameters": 2048, |
| "shape": [ |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.1.self_attn.k_norm.weight", |
| "parameters": 128, |
| "shape": [ |
| 128 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.1.self_attn.k_proj.weight", |
| "parameters": 1048576, |
| "shape": [ |
| 512, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.1.self_attn.o_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 2048, |
| 4096 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.1.self_attn.q_norm.weight", |
| "parameters": 128, |
| "shape": [ |
| 128 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.1.self_attn.q_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 4096, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.1.self_attn.v_proj.weight", |
| "parameters": 1048576, |
| "shape": [ |
| 512, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.2.input_layernorm.weight", |
| "parameters": 2048, |
| "shape": [ |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.2.mlp.down_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 2048, |
| 768 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.2.mlp.gate_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 768, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.2.mlp.up_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 768, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.2.post_attention_layernorm.weight", |
| "parameters": 2048, |
| "shape": [ |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.2.self_attn.k_norm.weight", |
| "parameters": 128, |
| "shape": [ |
| 128 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.2.self_attn.k_proj.weight", |
| "parameters": 1048576, |
| "shape": [ |
| 512, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.2.self_attn.o_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 2048, |
| 4096 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.2.self_attn.q_norm.weight", |
| "parameters": 128, |
| "shape": [ |
| 128 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.2.self_attn.q_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 4096, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.2.self_attn.v_proj.weight", |
| "parameters": 1048576, |
| "shape": [ |
| 512, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.3.input_layernorm.weight", |
| "parameters": 2048, |
| "shape": [ |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.3.mlp.down_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 2048, |
| 768 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.3.mlp.gate_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 768, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.3.mlp.up_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 768, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.3.post_attention_layernorm.weight", |
| "parameters": 2048, |
| "shape": [ |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.3.self_attn.k_norm.weight", |
| "parameters": 128, |
| "shape": [ |
| 128 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.3.self_attn.k_proj.weight", |
| "parameters": 1048576, |
| "shape": [ |
| 512, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.3.self_attn.o_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 2048, |
| 4096 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.3.self_attn.q_norm.weight", |
| "parameters": 128, |
| "shape": [ |
| 128 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.3.self_attn.q_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 4096, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.3.self_attn.v_proj.weight", |
| "parameters": 1048576, |
| "shape": [ |
| 512, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.4.input_layernorm.weight", |
| "parameters": 2048, |
| "shape": [ |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.4.mlp.down_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 2048, |
| 768 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.4.mlp.gate_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 768, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.4.mlp.up_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 768, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.4.post_attention_layernorm.weight", |
| "parameters": 2048, |
| "shape": [ |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.4.self_attn.k_norm.weight", |
| "parameters": 128, |
| "shape": [ |
| 128 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.4.self_attn.k_proj.weight", |
| "parameters": 1048576, |
| "shape": [ |
| 512, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.4.self_attn.o_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 2048, |
| 4096 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.4.self_attn.q_norm.weight", |
| "parameters": 128, |
| "shape": [ |
| 128 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.4.self_attn.q_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 4096, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.4.self_attn.v_proj.weight", |
| "parameters": 1048576, |
| "shape": [ |
| 512, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "lm_head.weight", |
| "parameters": 65536000, |
| "shape": [ |
| 32000, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "norm.weight", |
| "parameters": 2048, |
| "shape": [ |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BOOL", |
| "name": "t2d", |
| "parameters": 262144, |
| "shape": [ |
| 262144 |
| ] |
| } |
| ] |
| } |
|
|
