Upload folder using huggingface_hub

config.json

@@ -8,39 +8,38 @@
     "eos_token_id": 151643,
     "head_dim": 128,
     "hidden_act": "silu",
-    "hidden_size": 2048,
+    "hidden_size": 2560,
     "initializer_range": 0.02,
-    "intermediate_size": 6144,
+    "intermediate_size": 9728,
     "max_position_embeddings": 131072,
-    "max_window_layers": 24,
-    "mix_coeff": 1.0,
+    "max_window_layers": 36,
+    "mix_coeff": 0,
     "model_type": "qwen3",
-    "num_attention_heads": 16,
-    "num_hidden_layers": 24,
-    "num_key_value_heads": 16,
+    "num_attention_heads": 32,
+    "num_hidden_layers": 36,
+    "num_key_value_heads": 8,
     "rms_norm_eps": 1e-06,
     "rope_scaling": null,
-    "rope_theta": 10000,
+    "rope_theta": 1000000,
     "sliding_window": null,
     "tie_word_embeddings": true,
     "torch_dtype": "bfloat16",
     "transformers_version": "4.51.0",
     "use_cache": true,
     "use_sliding_window": false,
-    "vocab_size": 151936,
+    "vocab_size": 151937,
     "summary_token_begin": 151936,
     "summary_chunk_size": 8,
     "summary_token_num": 1,
     "use_summary_attention": true,
-    "summary_sliding_chunk_num": "([128]*3+[1024]*1)*6",
+    "summary_sliding_chunk_num": "([128]*3+[16768]*1)*9",
+    "summary_layer_freq": "([1])*36",
     "summary_chunk_position_ids_type": "origin",
     "summary_token_position_ids_type": "last_chunk_slice_right",
     "summary_independent_parameters": true,
     "summary_independent_attention_layernorm": false,
-    "summary_attention_mode": "kernel",
     "auto_map": {
-      "AutoModel": "modeling_qwen3sa.Qwen3Model",
-      "AutoModelForCausalLM": "modeling_qwen3sa.Qwen3ForCausalLM"
+      "AutoModel": "modeling_qwen3.Qwen3Model",
+      "AutoModelForCausalLM": "modeling_qwen3.Qwen3ForCausalLM"
     }
   }
-  

generation_config.json

@@ -1,7 +1,9 @@
 {
+  "_from_model_config": true,
   "bos_token_id": 151643,
-  "do_sample": false,
   "eos_token_id": 151643,
-  "max_new_tokens": 2048,
-  "transformers_version": "4.37.0"
-}
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.0.0",
+  "use_cache": true
+}