Upload folder using huggingface_hub

checkpoint-10000/config.json

@@ -0,0 +1,191 @@
+{
+  "architectures": [
+    "WeaverQwenForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "modeling_C3.WeaverConfig",
+    "AutoModel": "modeling_C3.WeaverQwenModel",
+    "AutoModelForCausalLM": "modeling_C3.WeaverQwenForCausalLM"
+  },
+  "bos_token_id": 151643,
+  "contexts_compression_llm_hidden_size": 1536,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "im_end_token": 151858,
+  "im_patch_token": 151859,
+  "im_start_token": 151857,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "latent_token_len": 32,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 36,
+  "model_type": "Weaver",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 36,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.55.4",
+  "use_cache": true,
+  "use_im_start_end": true,
+  "use_mrope": false,
+  "use_sliding_window": false,
+  "vocab_size": 151860,
+  "weaver_config": {
+    "_name_or_path": "/mnt/dolphinfs/ssd_pool/docker/user/hadoop-basecv-hl/hadoop-basecv/user/liufanfan/MM_out/C3_latent32 /checkpoint-1/llm1",
+    "add_cross_attention": false,
+    "architectures": [
+      "Qwen2ForCausalLM"
+    ],
+    "attention_dropout": 0.0,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": 151643,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": 151643,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "intermediate_size": 8960,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "max_position_embeddings": 32768,
+    "max_window_layers": 28,
+    "min_length": 0,
+    "model_type": "qwen2",
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 12,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 2,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "rms_norm_eps": 1e-06,
+    "rope_scaling": null,
+    "rope_theta": 1000000.0,
+    "sep_token_id": null,
+    "sliding_window": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "bfloat16",
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "use_cache": true,
+    "use_mrope": false,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  }
+}

checkpoint-10000/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "transformers_version": "4.55.4"
+}

checkpoint-10000/model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48ca527a27a3d04f9d9997c8754bf7020799a9b92781fba2bafa5ef4d2993285
+size 4957249008

checkpoint-10000/model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:423968ec255e710dd966c354c23977cf5d22204cf2460cc87265c546cf38ebbb
+size 4992407192

checkpoint-10000/model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:799aa1ccf9da1be799121d09e67eed0d0020b823fd00063c3557bfe6d5018e00
+size 121131992

checkpoint-10000/model.safetensors.index.json

@@ -0,0 +1,786 @@
+{
+  "metadata": {
+    "total_parameters": 5035347456,
+    "total_size": 10070694912
+  },
+  "weight_map": {
+    "model.Q.weight": "model-00002-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.35.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.35.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.35.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.memory_compressor.compressor.weight": "model-00002-of-00003.safetensors",
+    "model.memory_decompressor.up_gate.weight": "model-00002-of-00003.safetensors",
+    "model.memory_decompressor.up_val.weight": "model-00002-of-00003.safetensors",
+    "model.mm_projector.bias": "model-00002-of-00003.safetensors",
+    "model.mm_projector.weight": "model-00002-of-00003.safetensors",
+    "model.norm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.embed_tokens.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.26.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.26.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.26.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.26.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.26.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.26.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.26.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.26.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.26.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.3.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

checkpoint-10000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf43bedaf2d0fe8c496be3128e213a889646a746ca752b47b4c20a80e19e7b43
+size 2867444762

checkpoint-10000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4a9f217e852f439efa6bd32fde98d6867f11aa6ea13ddc021ba10af6a0b0934
+size 14645

checkpoint-10000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63ee0e358b0d1ea298abe717b65a5937359f1ec60985fcd2212a985daa2206c9
+size 1465

checkpoint-10000/special_tokens_map.json

@@ -0,0 +1,4 @@
+{
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<|endoftext|>"
+}

checkpoint-10000/tokenization_qwen.py

@@ -0,0 +1,264 @@
+# Copyright (c) Alibaba Cloud.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Tokenization classes for QWen."""
+
+import base64
+import logging
+import os
+import unicodedata
+from typing import Collection, Dict, List, Set, Tuple, Union
+
+import tiktoken
+from transformers import PreTrainedTokenizer, AddedToken
+
+logger = logging.getLogger(__name__)
+
+
+VOCAB_FILES_NAMES = {"vocab_file": "qwen.tiktoken"}
+
+PAT_STR = r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"""
+ENDOFTEXT = "<|endoftext|>"
+IMSTART = "<|im_start|>"
+IMEND = "<|im_end|>"
+# as the default behavior is changed to allow special tokens in
+# regular texts, the surface forms of special tokens need to be
+# as different as possible to minimize the impact
+EXTRAS = tuple((f"<|extra_{i}|>" for i in range(205)))
+SPECIAL_TOKENS = (
+    ENDOFTEXT,
+    IMSTART,
+    IMEND,
+) + EXTRAS
+
+
+def _load_tiktoken_bpe(tiktoken_bpe_file: str) -> Dict[bytes, int]:
+    with open(tiktoken_bpe_file, "rb") as f:
+        contents = f.read()
+    return {
+        base64.b64decode(token): int(rank)
+        for token, rank in (line.split() for line in contents.splitlines() if line)
+    }
+
+class QWenTokenizer(PreTrainedTokenizer):
+    """QWen tokenizer."""
+
+    vocab_files_names = VOCAB_FILES_NAMES
+
+    def __init__(
+        self,
+        vocab_file,
+        errors="replace",
+        image_start_tag='<img>',
+        image_end_tag='</img>',
+        image_pad_tag='<imgpad>',
+        ref_start_tag='<ref>',
+        ref_end_tag='</ref>',
+        box_start_tag='<box>',
+        box_end_tag='</box>',
+        quad_start_tag='<quad>',
+        quad_end_tag='</quad>',
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        
+        self.image_start_tag = image_start_tag
+        self.image_end_tag = image_end_tag
+        self.image_pad_tag = image_pad_tag
+        self.ref_start_tag = ref_start_tag
+        self.ref_end_tag = ref_end_tag
+        self.box_start_tag = box_start_tag
+        self.box_end_tag = box_end_tag
+        self.quad_start_tag = quad_start_tag
+        self.quad_end_tag = quad_end_tag
+        self.IMAGE_ST = (
+            ref_start_tag, ref_end_tag,
+            box_start_tag, box_end_tag,
+            quad_start_tag, quad_end_tag,
+            image_start_tag, image_end_tag,
+            image_pad_tag
+        )
+
+        self.errors = errors  # how to handle errors in decoding
+
+        self.mergeable_ranks = _load_tiktoken_bpe(vocab_file)  # type: dict[bytes, int]
+        self.special_tokens = {
+            token: index
+            for index, token in enumerate(
+                SPECIAL_TOKENS + self.IMAGE_ST, start=len(self.mergeable_ranks)
+            )
+        }
+        
+        self.img_start_id = self.special_tokens[self.image_start_tag]
+        self.img_end_id = self.special_tokens[self.image_end_tag]
+        self.img_pad_id = self.special_tokens[self.image_pad_tag]
+        self.ref_start_id = self.special_tokens[self.ref_start_tag]
+        self.ref_end_id = self.special_tokens[self.ref_end_tag]
+        self.box_start_id = self.special_tokens[self.box_start_tag]
+        self.box_end_id = self.special_tokens[self.box_end_tag]
+        self.quad_start_id = self.special_tokens[self.quad_start_tag]
+        self.quad_end_id = self.special_tokens[self.quad_end_tag]
+
+        enc = tiktoken.Encoding(
+            "Qwen",
+            pat_str=PAT_STR,
+            mergeable_ranks=self.mergeable_ranks,
+            special_tokens=self.special_tokens,
+        )
+        assert (
+            len(self.mergeable_ranks) + len(self.special_tokens) == enc.n_vocab
+        ), f"{len(self.mergeable_ranks) + len(self.special_tokens)} != {enc.n_vocab} in encoding"
+
+        self.decoder = {
+            v: k for k, v in self.mergeable_ranks.items()
+        }  # type: dict[int, bytes|str]
+        self.decoder.update({v: k for k, v in self.special_tokens.items()})
+
+        self.tokenizer = enc  # type: tiktoken.Encoding
+
+        self.eod_id = self.tokenizer.eot_token
+        self.im_start_id = self.special_tokens[IMSTART]
+        self.im_end_id = self.special_tokens[IMEND]
+
+    def __len__(self) -> int:
+        return self.tokenizer.n_vocab
+
+    def get_vocab(self) -> Dict[bytes, int]:
+        return self.mergeable_ranks
+
+    def convert_tokens_to_ids(
+        self, tokens: Union[bytes, str, List[Union[bytes, str]]]
+    ) -> List[int]:
+        ids = []
+        if isinstance(tokens, (str, bytes)):
+            if tokens in self.special_tokens:
+                return self.special_tokens[tokens]
+            else:
+                return self.mergeable_ranks.get(tokens)
+        for token in tokens:
+            if token in self.special_tokens:
+                ids.append(self.special_tokens[token])
+            else:
+                ids.append(self.mergeable_ranks.get(token))
+        return ids
+
+    def _add_tokens(self, new_tokens: Union[List[str], List[AddedToken]], special_tokens: bool = False) -> int:
+        if not special_tokens and new_tokens:
+            raise ValueError('Adding regular tokens is not supported')
+        for token in new_tokens:
+            surface_form = token.content if isinstance(token, AddedToken) else token
+            if surface_form not in SPECIAL_TOKENS:
+                raise ValueError('Adding unknown special tokens is not supported')
+        return 0
+
+    def save_vocabulary(self, save_directory: str, **kwargs) -> Tuple[str]:
+        """
+        Save only the vocabulary of the tokenizer (vocabulary).
+
+        Returns:
+            `Tuple(str)`: Paths to the files saved.
+        """
+        file_path = os.path.join(save_directory, "qwen.tiktoken")
+        with open(file_path, "w", encoding="utf8") as w:
+            for k, v in self.mergeable_ranks.items():
+                line = base64.b64encode(k).decode("utf8") + " " + str(v) + "\n"
+                w.write(line)
+        return (file_path,)
+
+    def tokenize(
+        self,
+        text: str,
+        allowed_special: Union[Set, str] = "all",
+        disallowed_special: Union[Collection, str] = (),
+        **kwargs,
+    ) -> List[Union[bytes, str]]:
+        """
+        Converts a string in a sequence of tokens.
+
+        Args:
+            text (`str`):
+                The sequence to be encoded.
+            allowed_special (`Literal["all"]` or `set`):
+                The surface forms of the tokens to be encoded as special tokens in regular texts.
+                Default to "all".
+            disallowed_special (`Literal["all"]` or `Collection`):
+                The surface forms of the tokens that should not be in regular texts and trigger errors.
+                Default to an empty tuple.
+
+            kwargs (additional keyword arguments, *optional*):
+                Will be passed to the underlying model specific encode method.
+
+        Returns:
+            `List[bytes|str]`: The list of tokens.
+        """
+        tokens = []
+        text = unicodedata.normalize("NFC", text)
+
+        # this implementation takes a detour: text -> token id -> token surface forms
+        for t in self.tokenizer.encode(
+            text, allowed_special=allowed_special, disallowed_special=disallowed_special
+        ):
+            tokens.append(self.decoder[t])
+        return tokens
+
+    def convert_tokens_to_string(self, tokens: List[Union[bytes, str]]) -> str:
+        """
+        Converts a sequence of tokens in a single string.
+        """
+        text = ""
+        temp = b""
+        for t in tokens:
+            if isinstance(t, str):
+                if temp:
+                    text += temp.decode("utf-8", errors=self.errors)
+                    temp = b""
+                text += t
+            elif isinstance(t, bytes):
+                temp += t
+            else:
+                raise TypeError("token should only be of type types or str")
+        if temp:
+            text += temp.decode("utf-8", errors=self.errors)
+        return text
+
+    @property
+    def vocab_size(self):
+        return self.tokenizer.n_vocab
+
+    def _convert_id_to_token(self, index: int) -> Union[bytes, str]:
+        """Converts an id to a token, special tokens included"""
+        if index in self.decoder:
+            return self.decoder[index]
+        raise ValueError("unknown ids")
+
+    def _convert_token_to_id(self, token: Union[bytes, str]) -> int:
+        """Converts a token to an id using the vocab, special tokens included"""
+        if token in self.special_tokens:
+            return self.special_tokens[token]
+        if token in self.mergeable_ranks:
+            return self.mergeable_ranks[token]
+        raise ValueError("unknown token")
+
+    def _tokenize(self, text: str, **kwargs):
+        """
+        Converts a string in a sequence of tokens (string), using the tokenizer. Split in words for word-based
+        vocabulary or sub-words for sub-word-based vocabularies (BPE/SentencePieces/WordPieces).
+
+        Do NOT take care of added tokens.
+        """
+        raise NotImplementedError
+
+    def _decode(
+        self,
+        token_ids: Union[int, List[int]],
+        skip_special_tokens: bool = False,
+        errors: str = None,
+        **kwargs,
+    ) -> str:
+        if isinstance(token_ids, int):
+            token_ids = [token_ids]
+        if skip_special_tokens:
+            token_ids = [i for i in token_ids if i < self.eod_id]
+        return self.tokenizer.decode(token_ids, errors=errors or self.errors)

checkpoint-10000/tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "added_tokens_decoder": {},
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_qwen.QWenTokenizer",
+      null
+    ]
+  },
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 81920,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "tokenizer_class": "QWenTokenizer"
+}

checkpoint-10000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48eeb98800955628b366418ce7d2c4fa58658223df723e1c82d379ebef337e2e
+size 5777

checkpoint-11721/config.json

@@ -0,0 +1,191 @@
+{
+  "architectures": [
+    "WeaverQwenForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "modeling_C3.WeaverConfig",
+    "AutoModel": "modeling_C3.WeaverQwenModel",
+    "AutoModelForCausalLM": "modeling_C3.WeaverQwenForCausalLM"
+  },
+  "bos_token_id": 151643,
+  "contexts_compression_llm_hidden_size": 1536,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "im_end_token": 151858,
+  "im_patch_token": 151859,
+  "im_start_token": 151857,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "latent_token_len": 32,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 36,
+  "model_type": "Weaver",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 36,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.55.4",
+  "use_cache": true,
+  "use_im_start_end": true,
+  "use_mrope": false,
+  "use_sliding_window": false,
+  "vocab_size": 151860,
+  "weaver_config": {
+    "_name_or_path": "/mnt/dolphinfs/ssd_pool/docker/user/hadoop-basecv-hl/hadoop-basecv/user/liufanfan/MM_out/C3_latent32 /checkpoint-1/llm1",
+    "add_cross_attention": false,
+    "architectures": [
+      "Qwen2ForCausalLM"
+    ],
+    "attention_dropout": 0.0,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": 151643,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": 151643,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "intermediate_size": 8960,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "max_position_embeddings": 32768,
+    "max_window_layers": 28,
+    "min_length": 0,
+    "model_type": "qwen2",
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 12,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 2,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "rms_norm_eps": 1e-06,
+    "rope_scaling": null,
+    "rope_theta": 1000000.0,
+    "sep_token_id": null,
+    "sliding_window": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "bfloat16",
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "use_cache": true,
+    "use_mrope": false,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  }
+}

checkpoint-11721/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "transformers_version": "4.55.4"
+}

checkpoint-11721/model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c71f1d5d40957a5842ce60a48930a1e09f076d480424c148df8523482b4028cc
+size 4957249008

checkpoint-11721/model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aeab06123b718473a4660ebff7c831a4a8a77ada458302599eeaf9a4e4b72adf
+size 4992407192

checkpoint-11721/model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:799aa1ccf9da1be799121d09e67eed0d0020b823fd00063c3557bfe6d5018e00
+size 121131992

checkpoint-11721/model.safetensors.index.json

@@ -0,0 +1,786 @@
+{
+  "metadata": {
+    "total_parameters": 5035347456,
+    "total_size": 10070694912
+  },
+  "weight_map": {
+    "model.Q.weight": "model-00002-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.35.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.35.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.35.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.memory_compressor.compressor.weight": "model-00002-of-00003.safetensors",
+    "model.memory_decompressor.up_gate.weight": "model-00002-of-00003.safetensors",
+    "model.memory_decompressor.up_val.weight": "model-00002-of-00003.safetensors",
+    "model.mm_projector.bias": "model-00002-of-00003.safetensors",
+    "model.mm_projector.weight": "model-00002-of-00003.safetensors",
+    "model.norm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.embed_tokens.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.26.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.26.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.26.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.26.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.26.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.26.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.26.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.26.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.26.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.3.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

checkpoint-11721/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:159a625f439a7a81227d47afec6c8d76d386a36ff8425a3a13491904005efb84
+size 2867444762

checkpoint-11721/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4a9f217e852f439efa6bd32fde98d6867f11aa6ea13ddc021ba10af6a0b0934
+size 14645

checkpoint-11721/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e56f8cb4dc909acefe414427aa0178fe1870460c071cbe059cd44a6d618c19d
+size 1465

checkpoint-11721/special_tokens_map.json

@@ -0,0 +1,4 @@
+{
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<|endoftext|>"
+}

checkpoint-11721/tokenization_qwen.py

@@ -0,0 +1,264 @@
+# Copyright (c) Alibaba Cloud.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Tokenization classes for QWen."""
+
+import base64
+import logging
+import os
+import unicodedata
+from typing import Collection, Dict, List, Set, Tuple, Union
+
+import tiktoken
+from transformers import PreTrainedTokenizer, AddedToken
+
+logger = logging.getLogger(__name__)
+
+
+VOCAB_FILES_NAMES = {"vocab_file": "qwen.tiktoken"}
+
+PAT_STR = r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"""
+ENDOFTEXT = "<|endoftext|>"
+IMSTART = "<|im_start|>"
+IMEND = "<|im_end|>"
+# as the default behavior is changed to allow special tokens in
+# regular texts, the surface forms of special tokens need to be
+# as different as possible to minimize the impact
+EXTRAS = tuple((f"<|extra_{i}|>" for i in range(205)))
+SPECIAL_TOKENS = (
+    ENDOFTEXT,
+    IMSTART,
+    IMEND,
+) + EXTRAS
+
+
+def _load_tiktoken_bpe(tiktoken_bpe_file: str) -> Dict[bytes, int]:
+    with open(tiktoken_bpe_file, "rb") as f:
+        contents = f.read()
+    return {
+        base64.b64decode(token): int(rank)
+        for token, rank in (line.split() for line in contents.splitlines() if line)
+    }
+
+class QWenTokenizer(PreTrainedTokenizer):
+    """QWen tokenizer."""
+
+    vocab_files_names = VOCAB_FILES_NAMES
+
+    def __init__(
+        self,
+        vocab_file,
+        errors="replace",
+        image_start_tag='<img>',
+        image_end_tag='</img>',
+        image_pad_tag='<imgpad>',
+        ref_start_tag='<ref>',
+        ref_end_tag='</ref>',
+        box_start_tag='<box>',
+        box_end_tag='</box>',
+        quad_start_tag='<quad>',
+        quad_end_tag='</quad>',
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        
+        self.image_start_tag = image_start_tag
+        self.image_end_tag = image_end_tag
+        self.image_pad_tag = image_pad_tag
+        self.ref_start_tag = ref_start_tag
+        self.ref_end_tag = ref_end_tag
+        self.box_start_tag = box_start_tag
+        self.box_end_tag = box_end_tag
+        self.quad_start_tag = quad_start_tag
+        self.quad_end_tag = quad_end_tag
+        self.IMAGE_ST = (
+            ref_start_tag, ref_end_tag,
+            box_start_tag, box_end_tag,
+            quad_start_tag, quad_end_tag,
+            image_start_tag, image_end_tag,
+            image_pad_tag
+        )
+
+        self.errors = errors  # how to handle errors in decoding
+
+        self.mergeable_ranks = _load_tiktoken_bpe(vocab_file)  # type: dict[bytes, int]
+        self.special_tokens = {
+            token: index
+            for index, token in enumerate(
+                SPECIAL_TOKENS + self.IMAGE_ST, start=len(self.mergeable_ranks)
+            )
+        }
+        
+        self.img_start_id = self.special_tokens[self.image_start_tag]
+        self.img_end_id = self.special_tokens[self.image_end_tag]
+        self.img_pad_id = self.special_tokens[self.image_pad_tag]
+        self.ref_start_id = self.special_tokens[self.ref_start_tag]
+        self.ref_end_id = self.special_tokens[self.ref_end_tag]
+        self.box_start_id = self.special_tokens[self.box_start_tag]
+        self.box_end_id = self.special_tokens[self.box_end_tag]
+        self.quad_start_id = self.special_tokens[self.quad_start_tag]
+        self.quad_end_id = self.special_tokens[self.quad_end_tag]
+
+        enc = tiktoken.Encoding(
+            "Qwen",
+            pat_str=PAT_STR,
+            mergeable_ranks=self.mergeable_ranks,
+            special_tokens=self.special_tokens,
+        )
+        assert (
+            len(self.mergeable_ranks) + len(self.special_tokens) == enc.n_vocab
+        ), f"{len(self.mergeable_ranks) + len(self.special_tokens)} != {enc.n_vocab} in encoding"
+
+        self.decoder = {
+            v: k for k, v in self.mergeable_ranks.items()
+        }  # type: dict[int, bytes|str]
+        self.decoder.update({v: k for k, v in self.special_tokens.items()})
+
+        self.tokenizer = enc  # type: tiktoken.Encoding
+
+        self.eod_id = self.tokenizer.eot_token
+        self.im_start_id = self.special_tokens[IMSTART]
+        self.im_end_id = self.special_tokens[IMEND]
+
+    def __len__(self) -> int:
+        return self.tokenizer.n_vocab
+
+    def get_vocab(self) -> Dict[bytes, int]:
+        return self.mergeable_ranks
+
+    def convert_tokens_to_ids(
+        self, tokens: Union[bytes, str, List[Union[bytes, str]]]
+    ) -> List[int]:
+        ids = []
+        if isinstance(tokens, (str, bytes)):
+            if tokens in self.special_tokens:
+                return self.special_tokens[tokens]
+            else:
+                return self.mergeable_ranks.get(tokens)
+        for token in tokens:
+            if token in self.special_tokens:
+                ids.append(self.special_tokens[token])
+            else:
+                ids.append(self.mergeable_ranks.get(token))
+        return ids
+
+    def _add_tokens(self, new_tokens: Union[List[str], List[AddedToken]], special_tokens: bool = False) -> int:
+        if not special_tokens and new_tokens:
+            raise ValueError('Adding regular tokens is not supported')
+        for token in new_tokens:
+            surface_form = token.content if isinstance(token, AddedToken) else token
+            if surface_form not in SPECIAL_TOKENS:
+                raise ValueError('Adding unknown special tokens is not supported')
+        return 0
+
+    def save_vocabulary(self, save_directory: str, **kwargs) -> Tuple[str]:
+        """
+        Save only the vocabulary of the tokenizer (vocabulary).
+
+        Returns:
+            `Tuple(str)`: Paths to the files saved.
+        """
+        file_path = os.path.join(save_directory, "qwen.tiktoken")
+        with open(file_path, "w", encoding="utf8") as w:
+            for k, v in self.mergeable_ranks.items():
+                line = base64.b64encode(k).decode("utf8") + " " + str(v) + "\n"
+                w.write(line)
+        return (file_path,)
+
+    def tokenize(
+        self,
+        text: str,
+        allowed_special: Union[Set, str] = "all",
+        disallowed_special: Union[Collection, str] = (),
+        **kwargs,
+    ) -> List[Union[bytes, str]]:
+        """
+        Converts a string in a sequence of tokens.
+
+        Args:
+            text (`str`):
+                The sequence to be encoded.
+            allowed_special (`Literal["all"]` or `set`):
+                The surface forms of the tokens to be encoded as special tokens in regular texts.
+                Default to "all".
+            disallowed_special (`Literal["all"]` or `Collection`):
+                The surface forms of the tokens that should not be in regular texts and trigger errors.
+                Default to an empty tuple.
+
+            kwargs (additional keyword arguments, *optional*):
+                Will be passed to the underlying model specific encode method.
+
+        Returns:
+            `List[bytes|str]`: The list of tokens.
+        """
+        tokens = []
+        text = unicodedata.normalize("NFC", text)
+
+        # this implementation takes a detour: text -> token id -> token surface forms
+        for t in self.tokenizer.encode(
+            text, allowed_special=allowed_special, disallowed_special=disallowed_special
+        ):
+            tokens.append(self.decoder[t])
+        return tokens
+
+    def convert_tokens_to_string(self, tokens: List[Union[bytes, str]]) -> str:
+        """
+        Converts a sequence of tokens in a single string.
+        """
+        text = ""
+        temp = b""
+        for t in tokens:
+            if isinstance(t, str):
+                if temp:
+                    text += temp.decode("utf-8", errors=self.errors)
+                    temp = b""
+                text += t
+            elif isinstance(t, bytes):
+                temp += t
+            else:
+                raise TypeError("token should only be of type types or str")
+        if temp:
+            text += temp.decode("utf-8", errors=self.errors)
+        return text
+
+    @property
+    def vocab_size(self):
+        return self.tokenizer.n_vocab
+
+    def _convert_id_to_token(self, index: int) -> Union[bytes, str]:
+        """Converts an id to a token, special tokens included"""
+        if index in self.decoder:
+            return self.decoder[index]
+        raise ValueError("unknown ids")
+
+    def _convert_token_to_id(self, token: Union[bytes, str]) -> int:
+        """Converts a token to an id using the vocab, special tokens included"""
+        if token in self.special_tokens:
+            return self.special_tokens[token]
+        if token in self.mergeable_ranks:
+            return self.mergeable_ranks[token]
+        raise ValueError("unknown token")
+
+    def _tokenize(self, text: str, **kwargs):
+        """
+        Converts a string in a sequence of tokens (string), using the tokenizer. Split in words for word-based
+        vocabulary or sub-words for sub-word-based vocabularies (BPE/SentencePieces/WordPieces).
+
+        Do NOT take care of added tokens.
+        """
+        raise NotImplementedError
+
+    def _decode(
+        self,
+        token_ids: Union[int, List[int]],
+        skip_special_tokens: bool = False,
+        errors: str = None,
+        **kwargs,
+    ) -> str:
+        if isinstance(token_ids, int):
+            token_ids = [token_ids]
+        if skip_special_tokens:
+            token_ids = [i for i in token_ids if i < self.eod_id]
+        return self.tokenizer.decode(token_ids, errors=errors or self.errors)

checkpoint-11721/tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "added_tokens_decoder": {},
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_qwen.QWenTokenizer",
+      null
+    ]
+  },
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 81920,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "tokenizer_class": "QWenTokenizer"
+}

checkpoint-11721/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48eeb98800955628b366418ce7d2c4fa58658223df723e1c82d379ebef337e2e
+size 5777

checkpoint-5000/config.json

@@ -0,0 +1,191 @@
+{
+  "architectures": [
+    "WeaverQwenForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "modeling_C3.WeaverConfig",
+    "AutoModel": "modeling_C3.WeaverQwenModel",
+    "AutoModelForCausalLM": "modeling_C3.WeaverQwenForCausalLM"
+  },
+  "bos_token_id": 151643,
+  "contexts_compression_llm_hidden_size": 1536,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "im_end_token": 151858,
+  "im_patch_token": 151859,
+  "im_start_token": 151857,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "latent_token_len": 32,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 36,
+  "model_type": "Weaver",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 36,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.55.4",
+  "use_cache": true,
+  "use_im_start_end": true,
+  "use_mrope": false,
+  "use_sliding_window": false,
+  "vocab_size": 151860,
+  "weaver_config": {
+    "_name_or_path": "/mnt/dolphinfs/ssd_pool/docker/user/hadoop-basecv-hl/hadoop-basecv/user/liufanfan/MM_out/C3_latent32 /checkpoint-1/llm1",
+    "add_cross_attention": false,
+    "architectures": [
+      "Qwen2ForCausalLM"
+    ],
+    "attention_dropout": 0.0,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": 151643,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": 151643,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "intermediate_size": 8960,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "max_position_embeddings": 32768,
+    "max_window_layers": 28,
+    "min_length": 0,
+    "model_type": "qwen2",
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 12,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 2,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "rms_norm_eps": 1e-06,
+    "rope_scaling": null,
+    "rope_theta": 1000000.0,
+    "sep_token_id": null,
+    "sliding_window": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "bfloat16",
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "use_cache": true,
+    "use_mrope": false,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  }
+}

checkpoint-5000/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "transformers_version": "4.55.4"
+}

checkpoint-5000/model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:712e6a9b0c15ee50a004ebbbcc691a69b009ec777524199bf462de9f75b6b22b
+size 4957249008

checkpoint-5000/model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18475e7df6e3237031169d9638572510644a12ee102e40fb1de1207fa8b44232
+size 4992407192

checkpoint-5000/model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:799aa1ccf9da1be799121d09e67eed0d0020b823fd00063c3557bfe6d5018e00
+size 121131992

checkpoint-5000/model.safetensors.index.json

@@ -0,0 +1,786 @@
+{
+  "metadata": {
+    "total_parameters": 5035347456,
+    "total_size": 10070694912
+  },
+  "weight_map": {
+    "model.Q.weight": "model-00002-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.35.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.35.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.35.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.memory_compressor.compressor.weight": "model-00002-of-00003.safetensors",
+    "model.memory_decompressor.up_gate.weight": "model-00002-of-00003.safetensors",
+    "model.memory_decompressor.up_val.weight": "model-00002-of-00003.safetensors",
+    "model.mm_projector.bias": "model-00002-of-00003.safetensors",
+    "model.mm_projector.weight": "model-00002-of-00003.safetensors",
+    "model.norm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.embed_tokens.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.0.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.1.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.10.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.2.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.24.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.25.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.26.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.26.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.26.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.26.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.26.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.26.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.26.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.26.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.26.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.weaver.model.layers.3.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.3.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.4.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.5.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.6.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.7.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.8.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.weaver.model.layers.9.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.weaver.model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

checkpoint-5000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92c79c42b60df9575291dfa92325ae04aaa5b0689e70d15f5d1df53e61ce9993
+size 2867444762

checkpoint-5000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:098b29492211804ab324a36f37466821d948280bb74fce4ba895c03f13ecd878
+size 14645

checkpoint-5000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:323a2a2b7a4a3ee1dd271ac1718f5bd0bd1ce504ffd707fa176536ee1a12d700
+size 1465

checkpoint-5000/special_tokens_map.json

@@ -0,0 +1,4 @@
+{
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<|endoftext|>"
+}

checkpoint-5000/tokenization_qwen.py

@@ -0,0 +1,264 @@
+# Copyright (c) Alibaba Cloud.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Tokenization classes for QWen."""
+
+import base64
+import logging
+import os
+import unicodedata
+from typing import Collection, Dict, List, Set, Tuple, Union
+
+import tiktoken
+from transformers import PreTrainedTokenizer, AddedToken
+
+logger = logging.getLogger(__name__)
+
+
+VOCAB_FILES_NAMES = {"vocab_file": "qwen.tiktoken"}
+
+PAT_STR = r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"""
+ENDOFTEXT = "<|endoftext|>"
+IMSTART = "<|im_start|>"
+IMEND = "<|im_end|>"
+# as the default behavior is changed to allow special tokens in
+# regular texts, the surface forms of special tokens need to be
+# as different as possible to minimize the impact
+EXTRAS = tuple((f"<|extra_{i}|>" for i in range(205)))
+SPECIAL_TOKENS = (
+    ENDOFTEXT,
+    IMSTART,
+    IMEND,
+) + EXTRAS
+
+
+def _load_tiktoken_bpe(tiktoken_bpe_file: str) -> Dict[bytes, int]:
+    with open(tiktoken_bpe_file, "rb") as f:
+        contents = f.read()
+    return {
+        base64.b64decode(token): int(rank)
+        for token, rank in (line.split() for line in contents.splitlines() if line)
+    }
+
+class QWenTokenizer(PreTrainedTokenizer):
+    """QWen tokenizer."""
+
+    vocab_files_names = VOCAB_FILES_NAMES
+
+    def __init__(
+        self,
+        vocab_file,
+        errors="replace",
+        image_start_tag='<img>',
+        image_end_tag='</img>',
+        image_pad_tag='<imgpad>',
+        ref_start_tag='<ref>',
+        ref_end_tag='</ref>',
+        box_start_tag='<box>',
+        box_end_tag='</box>',
+        quad_start_tag='<quad>',
+        quad_end_tag='</quad>',
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        
+        self.image_start_tag = image_start_tag
+        self.image_end_tag = image_end_tag
+        self.image_pad_tag = image_pad_tag
+        self.ref_start_tag = ref_start_tag
+        self.ref_end_tag = ref_end_tag
+        self.box_start_tag = box_start_tag
+        self.box_end_tag = box_end_tag
+        self.quad_start_tag = quad_start_tag
+        self.quad_end_tag = quad_end_tag
+        self.IMAGE_ST = (
+            ref_start_tag, ref_end_tag,
+            box_start_tag, box_end_tag,
+            quad_start_tag, quad_end_tag,
+            image_start_tag, image_end_tag,
+            image_pad_tag
+        )
+
+        self.errors = errors  # how to handle errors in decoding
+
+        self.mergeable_ranks = _load_tiktoken_bpe(vocab_file)  # type: dict[bytes, int]
+        self.special_tokens = {
+            token: index
+            for index, token in enumerate(
+                SPECIAL_TOKENS + self.IMAGE_ST, start=len(self.mergeable_ranks)
+            )
+        }
+        
+        self.img_start_id = self.special_tokens[self.image_start_tag]
+        self.img_end_id = self.special_tokens[self.image_end_tag]
+        self.img_pad_id = self.special_tokens[self.image_pad_tag]
+        self.ref_start_id = self.special_tokens[self.ref_start_tag]
+        self.ref_end_id = self.special_tokens[self.ref_end_tag]
+        self.box_start_id = self.special_tokens[self.box_start_tag]
+        self.box_end_id = self.special_tokens[self.box_end_tag]
+        self.quad_start_id = self.special_tokens[self.quad_start_tag]
+        self.quad_end_id = self.special_tokens[self.quad_end_tag]
+
+        enc = tiktoken.Encoding(
+            "Qwen",
+            pat_str=PAT_STR,
+            mergeable_ranks=self.mergeable_ranks,
+            special_tokens=self.special_tokens,
+        )
+        assert (
+            len(self.mergeable_ranks) + len(self.special_tokens) == enc.n_vocab
+        ), f"{len(self.mergeable_ranks) + len(self.special_tokens)} != {enc.n_vocab} in encoding"
+
+        self.decoder = {
+            v: k for k, v in self.mergeable_ranks.items()
+        }  # type: dict[int, bytes|str]
+        self.decoder.update({v: k for k, v in self.special_tokens.items()})
+
+        self.tokenizer = enc  # type: tiktoken.Encoding
+
+        self.eod_id = self.tokenizer.eot_token
+        self.im_start_id = self.special_tokens[IMSTART]
+        self.im_end_id = self.special_tokens[IMEND]
+
+    def __len__(self) -> int:
+        return self.tokenizer.n_vocab
+
+    def get_vocab(self) -> Dict[bytes, int]:
+        return self.mergeable_ranks
+
+    def convert_tokens_to_ids(
+        self, tokens: Union[bytes, str, List[Union[bytes, str]]]
+    ) -> List[int]:
+        ids = []
+        if isinstance(tokens, (str, bytes)):
+            if tokens in self.special_tokens:
+                return self.special_tokens[tokens]
+            else:
+                return self.mergeable_ranks.get(tokens)
+        for token in tokens:
+            if token in self.special_tokens:
+                ids.append(self.special_tokens[token])
+            else:
+                ids.append(self.mergeable_ranks.get(token))
+        return ids
+
+    def _add_tokens(self, new_tokens: Union[List[str], List[AddedToken]], special_tokens: bool = False) -> int:
+        if not special_tokens and new_tokens:
+            raise ValueError('Adding regular tokens is not supported')
+        for token in new_tokens:
+            surface_form = token.content if isinstance(token, AddedToken) else token
+            if surface_form not in SPECIAL_TOKENS:
+                raise ValueError('Adding unknown special tokens is not supported')
+        return 0
+
+    def save_vocabulary(self, save_directory: str, **kwargs) -> Tuple[str]:
+        """
+        Save only the vocabulary of the tokenizer (vocabulary).
+
+        Returns:
+            `Tuple(str)`: Paths to the files saved.
+        """
+        file_path = os.path.join(save_directory, "qwen.tiktoken")
+        with open(file_path, "w", encoding="utf8") as w:
+            for k, v in self.mergeable_ranks.items():
+                line = base64.b64encode(k).decode("utf8") + " " + str(v) + "\n"
+                w.write(line)
+        return (file_path,)
+
+    def tokenize(
+        self,
+        text: str,
+        allowed_special: Union[Set, str] = "all",
+        disallowed_special: Union[Collection, str] = (),
+        **kwargs,
+    ) -> List[Union[bytes, str]]:
+        """
+        Converts a string in a sequence of tokens.
+
+        Args:
+            text (`str`):
+                The sequence to be encoded.
+            allowed_special (`Literal["all"]` or `set`):
+                The surface forms of the tokens to be encoded as special tokens in regular texts.
+                Default to "all".
+            disallowed_special (`Literal["all"]` or `Collection`):
+                The surface forms of the tokens that should not be in regular texts and trigger errors.
+                Default to an empty tuple.
+
+            kwargs (additional keyword arguments, *optional*):
+                Will be passed to the underlying model specific encode method.
+
+        Returns:
+            `List[bytes|str]`: The list of tokens.
+        """
+        tokens = []
+        text = unicodedata.normalize("NFC", text)
+
+        # this implementation takes a detour: text -> token id -> token surface forms
+        for t in self.tokenizer.encode(
+            text, allowed_special=allowed_special, disallowed_special=disallowed_special
+        ):
+            tokens.append(self.decoder[t])
+        return tokens
+
+    def convert_tokens_to_string(self, tokens: List[Union[bytes, str]]) -> str:
+        """
+        Converts a sequence of tokens in a single string.
+        """
+        text = ""
+        temp = b""
+        for t in tokens:
+            if isinstance(t, str):
+                if temp:
+                    text += temp.decode("utf-8", errors=self.errors)
+                    temp = b""
+                text += t
+            elif isinstance(t, bytes):
+                temp += t
+            else:
+                raise TypeError("token should only be of type types or str")
+        if temp:
+            text += temp.decode("utf-8", errors=self.errors)
+        return text
+
+    @property
+    def vocab_size(self):
+        return self.tokenizer.n_vocab
+
+    def _convert_id_to_token(self, index: int) -> Union[bytes, str]:
+        """Converts an id to a token, special tokens included"""
+        if index in self.decoder:
+            return self.decoder[index]
+        raise ValueError("unknown ids")
+
+    def _convert_token_to_id(self, token: Union[bytes, str]) -> int:
+        """Converts a token to an id using the vocab, special tokens included"""
+        if token in self.special_tokens:
+            return self.special_tokens[token]
+        if token in self.mergeable_ranks:
+            return self.mergeable_ranks[token]
+        raise ValueError("unknown token")
+
+    def _tokenize(self, text: str, **kwargs):
+        """
+        Converts a string in a sequence of tokens (string), using the tokenizer. Split in words for word-based
+        vocabulary or sub-words for sub-word-based vocabularies (BPE/SentencePieces/WordPieces).
+
+        Do NOT take care of added tokens.
+        """
+        raise NotImplementedError
+
+    def _decode(
+        self,
+        token_ids: Union[int, List[int]],
+        skip_special_tokens: bool = False,
+        errors: str = None,
+        **kwargs,
+    ) -> str:
+        if isinstance(token_ids, int):
+            token_ids = [token_ids]
+        if skip_special_tokens:
+            token_ids = [i for i in token_ids if i < self.eod_id]
+        return self.tokenizer.decode(token_ids, errors=errors or self.errors)

checkpoint-5000/tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "added_tokens_decoder": {},
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_qwen.QWenTokenizer",
+      null
+    ]
+  },
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 81920,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "tokenizer_class": "QWenTokenizer"
+}

checkpoint-5000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48eeb98800955628b366418ce7d2c4fa58658223df723e1c82d379ebef337e2e
+size 5777

config.json

@@ -0,0 +1,191 @@
+{
+  "architectures": [
+    "WeaverQwenForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "modeling_weaver.WeaverConfig",
+    "AutoModel": "modeling_weaver.WeaverQwenModel",
+    "AutoModelForCausalLM": "modeling_weaver.WeaverQwenForCausalLM"
+  },
+  "bos_token_id": 151643,
+  "contexts_compression_llm_hidden_size": 1536,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "im_end_token": 151858,
+  "im_patch_token": 151859,
+  "im_start_token": 151857,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "latent_token_len": 32,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 36,
+  "model_type": "Weaver",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 36,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.55.4",
+  "use_cache": true,
+  "use_im_start_end": true,
+  "use_mrope": false,
+  "use_sliding_window": false,
+  "vocab_size": 151860,
+  "weaver_config": {
+    "_name_or_path": "/mnt/dolphinfs/ssd_pool/docker/user/hadoop-basecv-hl/hadoop-basecv/user/liufanfan/MM_out/C3_latent32 /checkpoint-1/llm1",
+    "add_cross_attention": false,
+    "architectures": [
+      "Qwen2ForCausalLM"
+    ],
+    "attention_dropout": 0.0,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": 151643,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": 151643,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "intermediate_size": 8960,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "max_position_embeddings": 32768,
+    "max_window_layers": 28,
+    "min_length": 0,
+    "model_type": "qwen2",
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 12,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 2,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "rms_norm_eps": 1e-06,
+    "rope_scaling": null,
+    "rope_theta": 1000000.0,
+    "sep_token_id": null,
+    "sliding_window": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "bfloat16",
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "use_cache": true,
+    "use_mrope": false,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  }
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e77c717454ac411df4b7726ac3407fad0afa245d8f991025f540567c7b716db
+size 7605335368

modeling_weaver.py

@@ -0,0 +1,848 @@
+from transformers import AutoConfig, AutoModelForCausalLM, \
+                         Qwen2Config, Qwen2Model, Qwen2ForCausalLM, \
+                         CLIPVisionModel, CLIPImageProcessor
+from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
+from typing import List, Optional, Tuple, Union
+from transformers.cache_utils import Cache, DynamicCache
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.nn import CrossEntropyLoss
+import os
+
+import dataclasses
+from enum import auto, Enum
+from typing import List, Tuple
+from transformers import StoppingCriteria
+from transformers import TextStreamer
+
+class SeparatorStyle(Enum):
+    """Different separator style."""
+    SINGLE = auto()
+    TWO = auto()
+    MPT = auto()
+
+
+@dataclasses.dataclass
+class Conversation:
+    """A class that keeps all conversation history."""
+    system: str
+    roles: List[str]
+    messages: List[List[str]]
+    offset: int
+    sep_style: SeparatorStyle = SeparatorStyle.SINGLE
+    sep: str = "<|im_end|>"
+    sep2: str = None
+    version: str = "Unknown"
+
+    skip_next: bool = False
+
+    def get_prompt(self):
+        if self.sep_style == SeparatorStyle.SINGLE:
+            ret = self.system + self.sep + '\n'
+            for role, message in self.messages:
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    ret += role + ": " + message + self.sep
+                else:
+                    ret += role + ":"
+            return ret
+        elif self.sep_style == SeparatorStyle.TWO:
+            seps = [self.sep, self.sep2]
+            ret = self.system + seps[0]
+            for i, (role, message) in enumerate(self.messages):
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    ret += role + ": " + message + seps[i % 2]
+                else:
+                    ret += role + ":"
+            return ret
+        if self.sep_style == SeparatorStyle.MPT:
+            if self.system:
+                ret = self.system + self.sep 
+            else:
+                ret = ''
+            for role, message in self.messages:
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    ret += role + message + self.sep
+                else:
+                    ret += role
+            return ret
+        else:
+            raise ValueError(f"Invalid style: {self.sep_style}")
+
+    def append_message(self, role, message):
+        self.messages.append([role, message])
+
+    def get_images(self, return_pil=False):
+        images = []
+        for i, (role, msg) in enumerate(self.messages[self.offset:]):
+            if i % 2 == 0:
+                if type(msg) is tuple:
+                    import base64
+                    from io import BytesIO
+                    from PIL import Image
+                    msg, image, image_process_mode = msg
+                    if image_process_mode == "Pad":
+                        def expand2square(pil_img, background_color=(122, 116, 104)):
+                            width, height = pil_img.size
+                            if width == height:
+                                return pil_img
+                            elif width > height:
+                                result = Image.new(pil_img.mode, (width, width), background_color)
+                                # result.paste(pil_img, (0, (width - height) // 2))
+                                result.paste(pil_img)
+                                return result
+                            else:
+                                result = Image.new(pil_img.mode, (height, height), background_color)
+                                # result.paste(pil_img, ((height - width) // 2, 0))
+                                result.paste(pil_img)
+                                return result
+                        image = expand2square(image)
+                    elif image_process_mode == "Crop":
+                        max_hw, min_hw = max(image.size), min(image.size)
+                        aspect_ratio = max_hw / min_hw
+                        max_len, min_len = 800, 400
+                        shortest_edge = int(min(max_len / aspect_ratio, min_len, min_hw))
+                        longest_edge = int(shortest_edge * aspect_ratio)
+                        W, H = image.size
+                        if H > W:
+                            H, W = longest_edge, shortest_edge
+                        else:
+                            H, W = shortest_edge, longest_edge
+                        image = image.resize((W, H))
+                    elif image_process_mode == "Resize":
+                        image = image.resize((224, 224))
+                    else:
+                        raise ValueError(f"Invalid image_process_mode: {image_process_mode}")
+
+                    if return_pil:
+                        images.append(image)
+                    else:
+                        buffered = BytesIO()
+                        image.convert('RGB').save(buffered, format="JPEG")
+                        img_b64_str = base64.b64encode(buffered.getvalue()).decode()
+                        images.append(img_b64_str)
+        return images
+
+    def to_gradio_chatbot(self):
+        ret = []
+        for i, (role, msg) in enumerate(self.messages[self.offset:]):
+            if i % 2 == 0:
+                if type(msg) is tuple:
+                    import base64
+                    from io import BytesIO
+                    msg, image, image_process_mode = msg
+                    max_hw, min_hw = max(image.size), min(image.size)
+                    aspect_ratio = max_hw / min_hw
+                    max_len, min_len = 800, 400
+                    shortest_edge = int(min(max_len / aspect_ratio, min_len, min_hw))
+                    longest_edge = int(shortest_edge * aspect_ratio)
+                    W, H = image.size
+                    if H > W:
+                        H, W = longest_edge, shortest_edge
+                    else:
+                        H, W = shortest_edge, longest_edge
+                    image = image.resize((W, H))
+                    # image = image.resize((224, 224))
+                    buffered = BytesIO()
+                    image.save(buffered, format="JPEG")
+                    img_b64_str = base64.b64encode(buffered.getvalue()).decode()
+                    img_str = f'<img src="data:image/png;base64,{img_b64_str}" alt="user upload image" />'
+                    msg = msg.replace('<image>', img_str)
+                ret.append([msg, None])
+            else:
+                ret[-1][-1] = msg
+        return ret
+
+    def copy(self):
+        return Conversation(
+            system=self.system,
+            roles=self.roles,
+            messages=[[x, y] for x, y in self.messages],
+            offset=self.offset,
+            sep_style=self.sep_style,
+            sep=self.sep,
+            sep2=self.sep2)
+
+    def dict(self):
+        if len(self.get_images()) > 0:
+            return {
+                "system": self.system,
+                "roles": self.roles,
+                "messages": [[x, y[0] if type(y) is tuple else y] for x, y in self.messages],
+                "offset": self.offset,
+                "sep": self.sep,
+                "sep2": self.sep2,
+            }
+        return {
+            "system": self.system,
+            "roles": self.roles,
+            "messages": self.messages,
+            "offset": self.offset,
+            "sep": self.sep,
+            "sep2": self.sep2,
+        }
+
+
+conv_mpt = Conversation(
+    system="""<|im_start|>system
+You should follow the instructions carefully and explain your answers in detail.""",
+    # system = None,
+    roles=("<|im_start|>user\n", "<|im_start|>assistant\n"),
+    version="mpt",
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.MPT,
+    sep="<|im_end|>",
+)
+
+conv_templates = {
+  
+    "mpt": conv_mpt,
+  
+}
+
+
+class KeywordsStoppingCriteria(StoppingCriteria):
+    def __init__(self, keywords, tokenizer, input_ids):
+        self.keywords = keywords
+        self.keyword_ids = [tokenizer(keyword).input_ids for keyword in keywords]
+        self.keyword_ids = [keyword_id[0] for keyword_id in self.keyword_ids if type(keyword_id) is list and len(keyword_id) == 1]
+        self.tokenizer = tokenizer
+        self.start_len = None
+        self.input_ids = input_ids
+
+    def __call__(self, output_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
+        if self.start_len is None:
+            self.start_len = self.input_ids.shape[1]
+        else:
+            for keyword_id in self.keyword_ids:
+                if output_ids[0, -1] == keyword_id:
+                    return True
+            outputs = self.tokenizer.batch_decode(output_ids[:, self.start_len:], skip_special_tokens=True)[0]
+            for keyword in self.keywords:
+                if keyword in outputs:
+                    return True
+        return False
+
+
+DEFAULT_IMAGE_PATCH_TOKEN = '<imgpad>'
+DEFAULT_IM_START_TOKEN = '<img>'
+DEFAULT_IM_END_TOKEN = '</img>'
+
+class WeaverConfig(Qwen2Config):
+    model_type = "Weaver"
+
+# class MemoryCompressor(nn.Module):
+#     """
+#     使用 Cross-Attention 压缩 N 个 token 为 1 个 memory token
+#     可学习的 query 动态聚合最重要的信息
+#     """
+#     def __init__(self, hidden_size, num_latent_tokens=32, num_heads=8, dropout=0.1):
+#         super().__init__()
+#         self.hidden_size = hidden_size
+#         self.num_latent_tokens = num_latent_tokens
+        
+#         # 可学习的压缩 query（1 个 token 来"总结"所有输入）
+#         self.compress_query = nn.Parameter(torch.randn(1, 1, hidden_size) * 0.02)
+        
+#         # Cross-Attention
+#         self.cross_attn = nn.MultiheadAttention(
+#             embed_dim=hidden_size,
+#             num_heads=num_heads,
+#             dropout=dropout,
+#             batch_first=True
+#         )
+        
+#         # 信息瓶颈：进一步压缩
+#         bottleneck_dim = hidden_size // 2
+#         self.bottleneck = nn.Sequential(
+#             nn.Linear(hidden_size, bottleneck_dim),
+#             nn.SiLU(),
+#             nn.Linear(bottleneck_dim, hidden_size),
+#         )
+        
+#         # 层归一化
+#         self.norm1 = nn.LayerNorm(hidden_size)
+#         self.norm2 = nn.LayerNorm(hidden_size)
+        
+#         self._init_weights()
+    
+#     def _init_weights(self):
+#         """初始化为接近 mean pooling 的效果"""
+#         # Cross-attention 初始化为均匀注意力
+#         nn.init.xavier_uniform_(self.cross_attn.in_proj_weight)
+#         nn.init.zeros_(self.cross_attn.in_proj_bias)
+#         nn.init.xavier_uniform_(self.cross_attn.out_proj.weight)
+#         nn.init.zeros_(self.cross_attn.out_proj.bias)
+        
+#         # Bottleneck 初始化为近似恒等映射
+#         nn.init.xavier_uniform_(self.bottleneck[0].weight, gain=0.1)
+#         nn.init.zeros_(self.bottleneck[0].bias)
+#         nn.init.xavier_uniform_(self.bottleneck[2].weight, gain=0.1)
+#         nn.init.zeros_(self.bottleneck[2].bias)
+
+#     def forward(self, x, attention_mask=None):
+#         """
+#         Args:
+#             x: (B, N, H) - N 个 latent tokens
+#             attention_mask: (B, N) - 可选的 mask
+#         Returns:
+#             (B, 1, H) - 压缩后的单个 token
+#         """
+#         B = x.shape[0]
+        
+#         # 扩展 query 到 batch
+#         query = self.compress_query.expand(B, -1, -1)  # (B, 1, H)
+        
+#         # Cross-attention
+#         if attention_mask is not None:
+#             key_padding_mask = (attention_mask == 0)
+#         else:
+#             key_padding_mask = None
+        
+#         attended, attn_weights = self.cross_attn(
+#             query=query,
+#             key=x,
+#             value=x,
+#             key_padding_mask=key_padding_mask
+#         )  # (B, 1, H)
+        
+#         # 残差 + 归一化
+#         attended = self.norm1(attended + query)
+        
+#         # 信息瓶颈
+#         bottlenecked = self.bottleneck(attended)
+#         output = self.norm2(bottlenecked + attended)
+        
+#         return output # 返回 attention weights 用于可视化
+
+
+# class MemoryDecompressor(nn.Module):
+#     """
+#     使用位置编码 + 自注意力解压 1 个 token 为 N 个 token
+#     """
+#     def __init__(self, hidden_size, num_latent_tokens=32, num_heads=8, num_layers=2):
+#         super().__init__()
+#         self.hidden_size = hidden_size
+#         self.num_latent_tokens = num_latent_tokens
+        
+#         # 位置编码
+#         self.position_embed = nn.Embedding(num_latent_tokens, hidden_size)
+        
+#         # 初始展开投影
+#         self.expand_proj = nn.Linear(hidden_size, hidden_size)
+        
+#         # 自注意力精炼层
+#         self.refine_layers = nn.ModuleList([
+#             nn.TransformerEncoderLayer(
+#                 d_model=hidden_size,
+#                 nhead=num_heads,
+#                 dim_feedforward=hidden_size * 4,
+#                 dropout=0.1,
+#                 activation='gelu',
+#                 batch_first=True,
+#                 norm_first=True  # Pre-LN 更稳定
+#             )
+#             for _ in range(num_layers)
+#         ])
+        
+#         self.final_norm = nn.LayerNorm(hidden_size)
+        
+#         self._init_weights()
+    
+#     def _init_weights(self):
+#         """初始化为接近广播的效果"""
+#         nn.init.eye_(self.expand_proj.weight)
+#         nn.init.zeros_(self.expand_proj.bias)
+#         nn.init.normal_(self.position_embed.weight, std=0.02)
+
+#     def forward(self, x):
+#         """
+#         Args:
+#             x: (B, 1, H) - 压缩后的单个 token
+#         Returns:
+#             (B, N, H) - 解压后的 N 个 tokens
+#         """
+#         B = x.shape[0]
+#         device = x.device
+        
+#         # 广播到 N 个位置
+#         x_expanded = x.expand(-1, self.num_latent_tokens, -1)  # (B, N, H)
+        
+#         # 投影 + 位置编码
+#         positions = torch.arange(self.num_latent_tokens, device=device)
+#         pos_embed = self.position_embed(positions)  # (N, H)
+        
+#         hidden = self.expand_proj(x_expanded) + pos_embed  # (B, N, H)
+        
+#         # 自注意力精炼
+#         for layer in self.refine_layers:
+#             hidden = layer(hidden)
+        
+#         return self.final_norm(hidden)
+    
+
+class MemoryCompressor(nn.Module):
+    """
+    Weaver端：压缩 N 个 token 为 1 个 memory token
+    (B, N, H) -> (B, 1, H)
+    """
+    def __init__(self, hidden_size, num_latent_tokens=32):
+        super().__init__()
+        self.num_tokens = num_latent_tokens
+        self.hidden_size = hidden_size
+        self.input_dim = num_latent_tokens * hidden_size
+        
+        # Compress: N*H -> H
+        self.compressor = nn.Linear(self.input_dim, hidden_size, bias=False)
+
+    def forward(self, x):
+        # x: (Batch, N, H)
+        B, N, H = x.shape
+        x_flat = x.view(B, -1)  # (B, N*H)
+        compressed = self.compressor(x_flat)  # (B, H)
+        return compressed.unsqueeze(1)  # (B, 1, H)
+
+
+class MemoryDecompressor(nn.Module):
+    """
+    Reasoner端：解压 1 个 memory token 为 N 个 token
+    (B, 1, H) -> (B, N, H)
+    """
+    def __init__(self, hidden_size, num_latent_tokens=32):
+        super().__init__()
+        self.num_tokens = num_latent_tokens
+        self.hidden_size = hidden_size
+        self.output_dim = num_latent_tokens * hidden_size
+        
+        # Decompress (SwiGLU style): H -> N*H
+        self.up_gate = nn.Linear(hidden_size, self.output_dim, bias=False)
+        self.up_val = nn.Linear(hidden_size, self.output_dim, bias=False)
+        self.act_fn = nn.SiLU()
+
+    def forward(self, x):
+        # x: (Batch, 1, H)
+        B = x.shape[0]
+        x_squeezed = x.squeeze(1)  # (B, H)
+        
+        gate = self.act_fn(self.up_gate(x_squeezed))
+        val = self.up_val(x_squeezed)
+        out_flat = gate * val  # (B, N*H)
+        
+        return out_flat.view(B, self.num_tokens, self.hidden_size)  # (B, N, H)
+
+
+class WeaverQwenModel(Qwen2Model):
+    config_class = WeaverConfig
+
+    def __init__(self, config: Qwen2Config):
+        super(WeaverQwenModel, self).__init__(config)
+
+        self.Q = nn.Embedding(config.latent_token_len , config.contexts_compression_llm_hidden_size)
+        self.mm_projector = nn.Linear(config.contexts_compression_llm_hidden_size, config.hidden_size)
+        self.weaver = None
+        self.config.use_im_start_end = True
+        self.memory_compressor = MemoryCompressor(
+            hidden_size=config.hidden_size,
+            num_latent_tokens=config.latent_token_len
+        )
+        self.memory_decompressor = MemoryDecompressor(
+            hidden_size=config.hidden_size,
+            num_latent_tokens=config.latent_token_len
+        )
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        context_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        context_attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, BaseModelOutputWithPast]:
+
+        # HACK: replace back original embeddings for LLaVA pretraining
+        orig_embeds_params = getattr(self, 'orig_embeds_params', None)
+        if orig_embeds_params is not None:
+            with torch.no_grad():
+                self.get_input_embeddings().weight[:-self.num_new_tokens] = orig_embeds_params[:-self.num_new_tokens].data
+
+        if inputs_embeds is None:
+            inputs_embeds = self.embed_tokens(input_ids)
+
+        context_embeds = self.weaver.model.embed_tokens(context_ids)
+
+        #######encoder#######
+
+        if input_ids.shape[1] != 1 or self.training:
+            use_im_start_end = getattr(self.config, "use_im_start_end", -1)
+            im_patch_token = getattr(self.config, "im_patch_token", -1)
+            im_start_token = getattr(self.config, "im_start_token", -1)
+            im_end_token = getattr(self.config, "im_end_token", -1)
+            context_features = []
+
+            for i in range(context_embeds.shape[0]):
+                context_features.append([self.Q.weight])
+
+          
+            use_im_start_end = True
+            new_context_embeds = []
+            image_start_tokens_list = []
+            for cur_context_ids, cur_context_embeds, cur_context_features in zip(context_ids, context_embeds, context_features):
+
+                if use_im_start_end:
+                    image_start_tokens = torch.where(cur_context_ids == im_start_token)[0]
+                    image_start_tokens_list.append(image_start_tokens)
+
+                    for image_start_token_pos, per_cur_image_features in zip(image_start_tokens, cur_context_features):
+                        per_cur_image_features = per_cur_image_features.to(device=cur_context_embeds.device)
+                        num_patches = per_cur_image_features.shape[0]
+                        if cur_context_ids[image_start_token_pos + num_patches + 1] != im_end_token:
+                            raise ValueError("The image end token should follow the image start token.")
+                        
+                        cur_context_embeds = torch.cat(
+                            (
+                                cur_context_embeds[:image_start_token_pos+1], 
+                                per_cur_image_features, 
+                                cur_context_embeds[image_start_token_pos + num_patches + 1:]
+                            ), 
+                            dim=0
+                        )
+                    new_context_embeds.append(cur_context_embeds)
+                else:
+                    raise NotImplementedError
+
+            image_start_tokens_list = torch.tensor(image_start_tokens_list)
+
+            context_embeds = torch.stack(new_context_embeds, dim=0)
+            weaver_hidden_states = self.weaver.forward(
+                input_ids=None, attention_mask=context_attention_mask, past_key_values=None,
+                inputs_embeds=context_embeds, use_cache=None, position_ids = None,
+                output_attentions=output_attentions, output_hidden_states=True,
+                return_dict=return_dict
+            )['hidden_states'][-1]
+            latent_contexts = []
+            for i, weaver_hidden_state in enumerate(weaver_hidden_states): 
+                image_start_token_pos = image_start_tokens_list[i]
+                weaver_hidden_state = weaver_hidden_state[image_start_token_pos+1:image_start_token_pos + num_patches+1]
+                latent_contexts.append(weaver_hidden_state)
+           
+            ########decoder########
+            latent_features = []
+
+            for latent_context in latent_contexts:
+                aligned_context = self.mm_projector(latent_context)
+                
+                # Apply Bottleneck (Memory Formation)
+                # Input: (1, 32, H) -> Output: (1, 32, H)
+                compressed = self.memory_compressor(aligned_context.unsqueeze(0))
+                decompressed = self.memory_decompressor(compressed)
+                latent_features.append([decompressed.squeeze(0)])
+
+
+            new_input_embeds = []
+            for cur_input_ids, cur_input_embeds, cur_latent_features in zip(input_ids, inputs_embeds, latent_features):
+
+                if use_im_start_end:
+                    if (cur_input_ids == im_start_token).sum() != (cur_input_ids == im_end_token).sum():
+                        raise ValueError("The number of image start tokens and image end tokens should be the same.")
+                    image_start_tokens = torch.where(cur_input_ids == im_start_token)[0]
+                    for image_start_token_pos, per_cur_latent_features in zip(image_start_tokens, cur_latent_features):
+                        per_cur_latent_features = per_cur_latent_features.to(device=cur_input_embeds.device)
+                        num_patches = per_cur_latent_features.shape[0]
+                        if cur_input_ids[image_start_token_pos + num_patches + 1] != im_end_token:
+                            raise ValueError("The image end token should follow the image start token.")
+                        cur_input_embeds = torch.cat(
+                            (
+                                cur_input_embeds[:image_start_token_pos+1], 
+                                per_cur_latent_features, 
+                                cur_input_embeds[image_start_token_pos + num_patches + 1:]
+                            ), 
+                            dim=0
+                        )
+                    new_input_embeds.append(cur_input_embeds)
+                else:
+                    raise NotImplementedError
+
+            inputs_embeds = torch.stack(new_input_embeds, dim=0)
+            
+        return super(WeaverQwenModel, self).forward(
+            input_ids=None, attention_mask=attention_mask, past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds, use_cache=use_cache, position_ids = position_ids,
+            output_attentions=output_attentions, output_hidden_states=output_hidden_states,
+            return_dict=return_dict
+        )
+
+
+
+class WeaverQwenForCausalLM(Qwen2ForCausalLM):
+    config_class = WeaverConfig
+    # supports_gradient_checkpointing = True
+
+    def __init__(self, config):
+        super(Qwen2ForCausalLM, self).__init__(config)
+        self.model = WeaverQwenModel(config)
+
+        self.vocab_size = config.vocab_size
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_model(self):
+        return self.model
+
+    
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        context_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        context_attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        
+    ) -> Union[Tuple, CausalLMOutputWithPast]:
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+
+
+        outputs  = self.model(
+            input_ids=input_ids,
+            context_ids=context_ids,
+            past_key_values=past_key_values,
+            attention_mask=attention_mask,
+            context_attention_mask=context_attention_mask,
+            position_ids=position_ids,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict  
+        )
+
+
+        hidden_states = outputs[0]
+        logits = self.lm_head(hidden_states)
+        logits = logits.float()
+
+        # logits
+
+        loss = None
+        if labels is not None:
+            # Shift so that tokens < n predict n
+            shift_logits = logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+            # Flatten the tokens
+            loss_fct = CrossEntropyLoss()
+            shift_logits = shift_logits.view(-1, self.config.vocab_size)
+            shift_labels = shift_labels.view(-1)
+            # Enable model parallelism
+            shift_labels = shift_labels.to(shift_logits.device)
+            loss = loss_fct(shift_logits, shift_labels)
+
+        if not return_dict:
+            output = (logits,) + outputs[1:]
+            return (loss,) + output if loss is not None else output
+
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+
+    def prepare_inputs_for_generation(
+        self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs
+    ):
+        # Omit tokens covered by past_key_values
+        if past_key_values is not None:
+            if isinstance(past_key_values, Cache):
+                cache_length = past_key_values.get_seq_length()
+                past_length = past_key_values.get_seq_length()
+                #max_cache_length = past_key_values.get_max_length()
+                max_cache_length = None
+            else:
+                cache_length = past_length = past_key_values[0][0].shape[2]
+                max_cache_length = None
+
+            # Keep only the unprocessed tokens:
+            # 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where
+            # some of the inputs are exclusively passed as part of the cache (e.g. when passing input_embeds as
+            # input)
+            if attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]:
+                input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :]
+            # 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard
+            # input_ids based on the past_length.
+            elif past_length < input_ids.shape[1]:
+                input_ids = input_ids[:, past_length:]
+            # 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens.
+
+            # If we are about to go beyond the maximum cache length, we need to crop the input attention mask.
+            if (
+                max_cache_length is not None
+                and attention_mask is not None
+                and cache_length + input_ids.shape[1] > max_cache_length
+            ):
+                attention_mask = attention_mask[:, -max_cache_length:]
+
+        position_ids = kwargs.get("position_ids", None)
+        if attention_mask is not None and position_ids is None:
+            # create position_ids on the fly for batch generation
+            position_ids = attention_mask.long().cumsum(-1) - 1
+            position_ids.masked_fill_(attention_mask == 0, 1)
+            if past_key_values:
+                position_ids = position_ids[:, -input_ids.shape[1] :]
+
+        # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
+        if inputs_embeds is not None and past_key_values is None:
+            model_inputs = {"inputs_embeds": inputs_embeds}
+        else:
+            model_inputs = {"input_ids": input_ids}
+
+        model_inputs.update(
+            {
+                "position_ids": position_ids,
+                "past_key_values": past_key_values,
+                "use_cache": kwargs.get("use_cache"),
+                "attention_mask": attention_mask,
+                #"images": kwargs.get("images", None),
+                "context_ids": kwargs.get("context_ids", None),
+            }
+        )
+        return model_inputs
+
+    @classmethod
+    def from_pretrained(
+        cls,
+        pretrained_model_name_or_path,
+        *model_args,
+        **kwargs,
+    ):
+       
+        model = super().from_pretrained(
+            pretrained_model_name_or_path, *model_args, **kwargs
+        )
+        
+
+        if os.path.exists(pretrained_model_name_or_path):
+            weaver_path = os.path.join(pretrained_model_name_or_path, "weaver") 
+            print(f"Loading weaver from path: {weaver_path}")
+            
+            dtype = kwargs.get("torch_dtype", torch.float16) 
+            device = kwargs.get("device_map", "auto") 
+            
+            weaver = Qwen2ForCausalLM.from_pretrained(
+                weaver_path,
+                use_safetensors=kwargs.get("use_safetensors", True),
+                torch_dtype=dtype,
+                device_map=device, 
+            )
+
+        else:
+            
+            print(f"Loading weaver from HF")
+            
+            dtype = kwargs.get("torch_dtype", torch.float16) 
+            device = kwargs.get("device_map", "auto") 
+            
+            weaver = Qwen2ForCausalLM.from_pretrained(
+                pretrained_model_name_or_path,
+                subfolder="weaver",
+                use_safetensors=kwargs.get("use_safetensors", True),
+                torch_dtype=dtype,
+                device_map=device, 
+            )
+        
+        model.model.weaver = weaver
+        print("Successfully loaded and attached weaver.")
+    
+        
+        return model
+
+    def initialize_special_tokenizer(
+        self, 
+        tokenizer, 
+        device="cuda"
+    ):
+        config = self.get_model().config
+        self.resize_token_embeddings(len(tokenizer))
+        config.im_patch_token = tokenizer.convert_tokens_to_ids([DEFAULT_IMAGE_PATCH_TOKEN])[0]
+        config.use_im_start_end = True
+
+        if config.use_im_start_end:
+            self.resize_token_embeddings(len(tokenizer))
+            config.im_start_token, config.im_end_token = tokenizer.convert_tokens_to_ids([DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN])
+
+    def chat(self, tokenizer, context, prompt):
+
+        self.initialize_special_tokenizer(tokenizer)
+
+        qs = prompt
+        qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_PATCH_TOKEN*self.get_model().config.latent_token_len + DEFAULT_IM_END_TOKEN + '\n' + qs 
+        
+
+        context = context + DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_PATCH_TOKEN*self.get_model().config.latent_token_len + DEFAULT_IM_END_TOKEN
+
+        conv_mode = "mpt"
+        
+        conv = conv_templates[conv_mode].copy()
+        conv.append_message(conv.roles[0], qs)
+        conv.append_message(conv.roles[1], None)
+        prompt = conv.get_prompt()
+        inputs = tokenizer([prompt])
+        inputs_context = tokenizer([context])
+        input_ids = torch.as_tensor(inputs.input_ids).cuda()
+        inputs_context_ids = torch.as_tensor(inputs_context.input_ids).cuda()
+
+        stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
+        keywords = [stop_str]
+        stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids)
+        streamer = TextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
+
+
+        with torch.autocast("cuda", dtype=torch.bfloat16):
+            output_ids = self.generate(
+                input_ids,
+                context_ids=inputs_context_ids,
+                do_sample=False,
+                num_beams = 1,
+                no_repeat_ngram_size = 20,
+                streamer=streamer,
+                max_new_tokens=4096,
+                stopping_criteria=[stopping_criteria]
+                )
+
+            outputs = tokenizer.decode(output_ids[0, input_ids.shape[1]:]).strip()
+            
+            if outputs.endswith(stop_str):
+                outputs = outputs[:-len(stop_str)]
+            outputs = outputs.strip()
+        return outputs
+
+AutoConfig.register("Weaver", WeaverConfig)
+AutoModelForCausalLM.register(WeaverConfig, WeaverQwenForCausalLM)
+

special_tokens_map.json

@@ -0,0 +1,4 @@
+{
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<|endoftext|>"
+}