Add selected SFT variants S1-S4

.gitattributes

@@ -42,3 +42,7 @@ data/f3/output/scicode_sft_swift_F3_clean_v2.jsonl filter=lfs diff=lfs merge=lfs
 data/pretrain/code_messages_pre_train_filtered_stage1.jsonl filter=lfs diff=lfs merge=lfs -text
 data/f3/input/scicode_sft_swift_F3_clean.jsonl filter=lfs diff=lfs merge=lfs -text
 models/pretrained/qwen25_coder_7b_cpt/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+models/sft/sft_f3_clean_instruct/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+models/sft/sft_f3_hq_instruct/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+models/sft/sft_f3f4_instruct/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+models/sft/sft_f4_instruct/tokenizer.json filter=lfs diff=lfs merge=lfs -text

MANIFEST.json

@@ -1,6 +1,6 @@
 {
-  "schema_version": "milestone-2-model-training-artifacts-v2",
-  "generated_utc": "2026-08-23T07:38:32+00:00",
+  "schema_version": "milestone-2-model-training-artifacts-v3",
+  "generated_utc": "2026-08-23T08:25:48+00:00",
   "delivery_root": "/raid/data/weifeng/MileStone_2",
   "headline": {
     "models": [
@@ -8,8 +8,18 @@
       "models/pretrained/gpt2_124m_code_5epoch",
       "models/pretrained/qwen25_coder_7b_cpt",
       "models/pretrained/stage1_cpt",
-      "models/sft/sft_f3_refined_instruct"
+      "models/sft/sft_f3_refined_instruct",
+      "models/sft/sft_f3_clean_instruct",
+      "models/sft/sft_f3_hq_instruct",
+      "models/sft/sft_f3f4_instruct",
+      "models/sft/sft_f4_instruct"
     ],
+    "selected_sft_variants": {
+      "S1": "models/sft/sft_f3_clean_instruct",
+      "S2": "models/sft/sft_f3_hq_instruct",
+      "S3": "models/sft/sft_f3f4_instruct",
+      "S4": "models/sft/sft_f4_instruct"
+    },
     "reported_pass_at_1_pct": 9.1,
     "reported_pass_at_5_pct": 13.0,
     "evaluation": "Runnable alignment benchmark; 200 tasks; 5 samples/task; 1000 attempts"
@@ -19,20 +29,22 @@
     "raw llm.c binary checkpoints and distributed optimizer states are not included",
     "the 20,021,101,140-byte raw dataset used by the three added CPT models is not duplicated",
     "the exact May 0.5B training environment snapshot was not captured",
+    "S2-S4 datasets and variant-specific launch scripts remain in local candidate staging",
+    "Magicoder experiments are excluded by delivery-owner decision",
     "CodeGPT-Multilingual external redistribution requires rights confirmation"
   ],
   "storage": {
-    "hardlinked_bytes": 14098848389,
-    "copied_or_authored_bytes": 15246019585,
+    "hardlinked_bytes": 18097844981,
+    "copied_or_authored_bytes": 15246026180,
     "note": "Hard links save local disk; tar/zip/upload reads complete file bytes."
   },
-  "file_count": 138,
+  "file_count": 171,
   "files": [
     {
       "path": "README.md",
       "role": "README.md",
-      "bytes": 4757,
-      "sha256": "940c1e59ae7a979993baae5210e23c46be80066d8d3bd50b906204633aba6926",
+      "bytes": 6188,
+      "sha256": "1158cfe52c727407545277df8e053a88b4ff634db6975f2d947c0fc9a80548e2",
       "mode": "authored",
       "source": null,
       "source_sha256_matches": null
@@ -633,8 +645,8 @@
     {
       "path": "evidence/smoke_test_results.json",
       "role": "evidence",
-      "bytes": 2409,
-      "sha256": "06dcd8ad990e189445f3d964c2defedabd2351a6b617314ca4b87753cc29fe9c",
+      "bytes": 4114,
+      "sha256": "4d6943fd6ac7a96437da19b5c4b4ddd83746f78ee6547f7dca690391be7804fe",
       "mode": "authored",
       "source": null,
       "source_sha256_matches": null
@@ -1125,6 +1137,159 @@
       "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_curriculum_filtered_stage1/v1-20260514-232114/checkpoint-28175/trainer_state.json",
       "source_sha256_matches": true
     },
+    {
+      "path": "models/sft/README.md",
+      "role": "models",
+      "bytes": 1992,
+      "sha256": "cc0ab6ba18740855a558b91c686d7514c83cb2fe1e9a7f0dbfa2f398ef69a72d",
+      "mode": "authored",
+      "source": null,
+      "source_sha256_matches": null
+    },
+    {
+      "path": "models/sft/sft_f3_clean_instruct/args.json",
+      "role": "models",
+      "bytes": 39677,
+      "sha256": "77cbd14b2b1a955021b09903403339bddee49a5fe5c19d1149e984b1fdb213fd",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_clean_1epoch_1gpu/v1-20260505-184601/checkpoint-14550/args.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f3_clean_instruct/chat_template.jinja",
+      "role": "models",
+      "bytes": 2507,
+      "sha256": "cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_clean_1epoch_1gpu/v1-20260505-184601/checkpoint-14550/chat_template.jinja",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f3_clean_instruct/config.json",
+      "role": "models",
+      "bytes": 1284,
+      "sha256": "2a0bbab03998171855260860fbab81152ec51e67c7d69b85f32c7f624341fb0e",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_clean_1epoch_1gpu/v1-20260505-184601/checkpoint-14550/config.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f3_clean_instruct/generation_config.json",
+      "role": "models",
+      "bytes": 242,
+      "sha256": "87c51d61bd70a204993da36384296ccfeb56d46ee2cc7247929f917d0d8118f7",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_clean_1epoch_1gpu/v1-20260505-184601/checkpoint-14550/generation_config.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f3_clean_instruct/model.safetensors",
+      "role": "models",
+      "bytes": 988097824,
+      "sha256": "b0c56726f984cdb0ff6269ea300651a2a3e73c2827d07b36eccda22e2c6593eb",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_clean_1epoch_1gpu/v1-20260505-184601/checkpoint-14550/model.safetensors",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f3_clean_instruct/tokenizer.json",
+      "role": "models",
+      "bytes": 11421892,
+      "sha256": "3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_clean_1epoch_1gpu/v1-20260505-184601/checkpoint-14550/tokenizer.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f3_clean_instruct/tokenizer_config.json",
+      "role": "models",
+      "bytes": 692,
+      "sha256": "4d8ad3fe9729e37338fae50bac94ff03cb915071eb886b937077114e874dfa06",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_clean_1epoch_1gpu/v1-20260505-184601/checkpoint-14550/tokenizer_config.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f3_clean_instruct/trainer_state.json",
+      "role": "models",
+      "bytes": 313556,
+      "sha256": "2e305da69a4ebf87c10d3945b684cfcd1f18fbfee04fa32ce92ce0a6b60b2354",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_clean_1epoch_1gpu/v1-20260505-184601/checkpoint-14550/trainer_state.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f3_hq_instruct/args.json",
+      "role": "models",
+      "bytes": 39662,
+      "sha256": "25791c8c6904fb5ad1459161de30caff274709dad969e17f5572fcda45fd81b1",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v4_1epoch_1gpu/v0-20260519-205734/checkpoint-4212/args.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f3_hq_instruct/chat_template.jinja",
+      "role": "models",
+      "bytes": 2507,
+      "sha256": "cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v4_1epoch_1gpu/v0-20260519-205734/checkpoint-4212/chat_template.jinja",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f3_hq_instruct/config.json",
+      "role": "models",
+      "bytes": 1284,
+      "sha256": "2a0bbab03998171855260860fbab81152ec51e67c7d69b85f32c7f624341fb0e",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v4_1epoch_1gpu/v0-20260519-205734/checkpoint-4212/config.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f3_hq_instruct/generation_config.json",
+      "role": "models",
+      "bytes": 242,
+      "sha256": "87c51d61bd70a204993da36384296ccfeb56d46ee2cc7247929f917d0d8118f7",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v4_1epoch_1gpu/v0-20260519-205734/checkpoint-4212/generation_config.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f3_hq_instruct/model.safetensors",
+      "role": "models",
+      "bytes": 988097824,
+      "sha256": "ac9c2e900b1cb9931417a77be5d8dc989d1d83222bcc359984690527ce21b621",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v4_1epoch_1gpu/v0-20260519-205734/checkpoint-4212/model.safetensors",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f3_hq_instruct/tokenizer.json",
+      "role": "models",
+      "bytes": 11421892,
+      "sha256": "3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v4_1epoch_1gpu/v0-20260519-205734/checkpoint-4212/tokenizer.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f3_hq_instruct/tokenizer_config.json",
+      "role": "models",
+      "bytes": 692,
+      "sha256": "4d8ad3fe9729e37338fae50bac94ff03cb915071eb886b937077114e874dfa06",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v4_1epoch_1gpu/v0-20260519-205734/checkpoint-4212/tokenizer_config.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f3_hq_instruct/trainer_state.json",
+      "role": "models",
+      "bytes": 91265,
+      "sha256": "2af634bb210f1a7d2ff5228364967f16a044416847d29c8b77be0221e5752d58",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v4_1epoch_1gpu/v0-20260519-205734/checkpoint-4212/trainer_state.json",
+      "source_sha256_matches": true
+    },
     {
       "path": "models/sft/sft_f3_refined_instruct/args.json",
       "role": "models",
@@ -1197,11 +1362,155 @@
       "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v2_1epoch_1gpu/v0-20260508-164342/checkpoint-6435/trainer_state.json",
       "source_sha256_matches": true
     },
+    {
+      "path": "models/sft/sft_f3f4_instruct/args.json",
+      "role": "models",
+      "bytes": 39686,
+      "sha256": "cbfd1a1eadf3449c55fd8d7c4bc87941ba158f7234efaabac72bd970b520686c",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3F4_clean_1epoch_1gpu/v0-20260506-103602/checkpoint-15033/args.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f3f4_instruct/chat_template.jinja",
+      "role": "models",
+      "bytes": 2507,
+      "sha256": "cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3F4_clean_1epoch_1gpu/v0-20260506-103602/checkpoint-15033/chat_template.jinja",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f3f4_instruct/config.json",
+      "role": "models",
+      "bytes": 1284,
+      "sha256": "2a0bbab03998171855260860fbab81152ec51e67c7d69b85f32c7f624341fb0e",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3F4_clean_1epoch_1gpu/v0-20260506-103602/checkpoint-15033/config.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f3f4_instruct/generation_config.json",
+      "role": "models",
+      "bytes": 242,
+      "sha256": "87c51d61bd70a204993da36384296ccfeb56d46ee2cc7247929f917d0d8118f7",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3F4_clean_1epoch_1gpu/v0-20260506-103602/checkpoint-15033/generation_config.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f3f4_instruct/model.safetensors",
+      "role": "models",
+      "bytes": 988097824,
+      "sha256": "3fc57c00fa0e875ea148c69d11a87a1af3f9b4b6571f2fdbeeaa17ba4220f373",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3F4_clean_1epoch_1gpu/v0-20260506-103602/checkpoint-15033/model.safetensors",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f3f4_instruct/tokenizer.json",
+      "role": "models",
+      "bytes": 11421892,
+      "sha256": "3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3F4_clean_1epoch_1gpu/v0-20260506-103602/checkpoint-15033/tokenizer.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f3f4_instruct/tokenizer_config.json",
+      "role": "models",
+      "bytes": 692,
+      "sha256": "4d8ad3fe9729e37338fae50bac94ff03cb915071eb886b937077114e874dfa06",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3F4_clean_1epoch_1gpu/v0-20260506-103602/checkpoint-15033/tokenizer_config.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f3f4_instruct/trainer_state.json",
+      "role": "models",
+      "bytes": 324033,
+      "sha256": "b95c19cafed50b446810a01b544e4a9e7643110e538b42e306fce3c3daf8d9a4",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3F4_clean_1epoch_1gpu/v0-20260506-103602/checkpoint-15033/trainer_state.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f4_instruct/args.json",
+      "role": "models",
+      "bytes": 39671,
+      "sha256": "6955cf0f31569ffa12ad689770e64cbd5901eb40ad49cc34acc4eafb3b26a48c",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F4_clean_1epoch_1gpu/v3-20260505-141059/checkpoint-484/args.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f4_instruct/chat_template.jinja",
+      "role": "models",
+      "bytes": 2507,
+      "sha256": "cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F4_clean_1epoch_1gpu/v3-20260505-141059/checkpoint-484/chat_template.jinja",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f4_instruct/config.json",
+      "role": "models",
+      "bytes": 1284,
+      "sha256": "2a0bbab03998171855260860fbab81152ec51e67c7d69b85f32c7f624341fb0e",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F4_clean_1epoch_1gpu/v3-20260505-141059/checkpoint-484/config.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f4_instruct/generation_config.json",
+      "role": "models",
+      "bytes": 242,
+      "sha256": "87c51d61bd70a204993da36384296ccfeb56d46ee2cc7247929f917d0d8118f7",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F4_clean_1epoch_1gpu/v3-20260505-141059/checkpoint-484/generation_config.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f4_instruct/model.safetensors",
+      "role": "models",
+      "bytes": 988097824,
+      "sha256": "d9a6cc732c33b8ee955f2766c7be98a91272603d1be38f51569bf13e1ac292b1",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F4_clean_1epoch_1gpu/v3-20260505-141059/checkpoint-484/model.safetensors",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f4_instruct/tokenizer.json",
+      "role": "models",
+      "bytes": 11421892,
+      "sha256": "3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F4_clean_1epoch_1gpu/v3-20260505-141059/checkpoint-484/tokenizer.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f4_instruct/tokenizer_config.json",
+      "role": "models",
+      "bytes": 692,
+      "sha256": "4d8ad3fe9729e37338fae50bac94ff03cb915071eb886b937077114e874dfa06",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F4_clean_1epoch_1gpu/v3-20260505-141059/checkpoint-484/tokenizer_config.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f4_instruct/trainer_state.json",
+      "role": "models",
+      "bytes": 11278,
+      "sha256": "606a03e67a9539fd2ba2232dc6a76e4a397623e124fb9404005eb08e0dddb02d",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F4_clean_1epoch_1gpu/v3-20260505-141059/checkpoint-484/trainer_state.json",
+      "source_sha256_matches": true
+    },
     {
       "path": "tools/build_manifest.py",
       "role": "tools",
-      "bytes": 9915,
-      "sha256": "6a0aa61077ee762b0a78ff8c5fc3c273f90ff61a96b3f3e4fa507695f28057c8",
+      "bytes": 11187,
+      "sha256": "099962f86157d5684e1d18a463c6a691cb5a3f0d8b8845688e23d243fad0352f",
       "mode": "authored",
       "source": null,
       "source_sha256_matches": null
@@ -1209,8 +1518,8 @@
     {
       "path": "tools/smoke_test.py",
       "role": "tools",
-      "bytes": 3902,
-      "sha256": "7d67ef5a41617dbf0e902c6c6d7b129c287cab50f4f08b150757dc396edf04cf",
+      "bytes": 4097,
+      "sha256": "7f2dc8c83cb06184eea06972a031840de2d256bf88a7242b1204408a8c49039d",
       "mode": "authored",
       "source": null,
       "source_sha256_matches": null

README.md

@@ -1,8 +1,9 @@
 # Milestone 2 Model and Training Artefact Delivery
 
-This English-language package consolidates five historical model checkpoints, the
-corresponding training and data-preparation code, launch scripts, provenance evidence,
-datasets already selected for delivery, and integrity tooling.
+This English-language package consolidates nine historical model checkpoints, the
+core training and data-preparation code, launch scripts, provenance evidence, datasets
+already selected for delivery, and integrity tooling. Four additional SFT checkpoints
+(S1--S4) were selected for this model-only increment.
 
 ## Delivered Models
 
@@ -13,11 +14,21 @@ datasets already selected for delivery, and integrity tooling.
 | Qwen2.5-Coder-7B CPT | `models/pretrained/qwen25_coder_7b_cpt/` | `Qwen/Qwen2.5-Coder-7B`, revision `0396a76181e127dfc13e5c5ec48a8cee09938b02` | full-parameter CPT, 3 epochs, 18,165/18,165 steps |
 | Qwen2.5-Coder-0.5B Stage-1 CPT | `models/pretrained/stage1_cpt/` | `Qwen/Qwen2.5-Coder-0.5B` | continued-pre-training deliverable |
 | Qwen2.5-Coder-0.5B-Instruct F3 SFT | `models/sft/sft_f3_refined_instruct/` | `Qwen/Qwen2.5-Coder-0.5B-Instruct` | reported pass@1 9.1%; pass@5 13.0% |
+| S1: Qwen2.5-Coder-0.5B-Instruct, F3 clean | `models/sft/sft_f3_clean_instruct/` | `Qwen/Qwen2.5-Coder-0.5B-Instruct` | 473,465 records; step 14,550; reported pass@1 9.1%, pass@5 13.0% |
+| S2: Qwen2.5-Coder-0.5B-Instruct, F3 high-quality v4 | `models/sft/sft_f3_hq_instruct/` | `Qwen/Qwen2.5-Coder-0.5B-Instruct` | 134,775 records; step 4,212; reported pass@1 8.3%, pass@5 13.0% |
+| S3: Qwen2.5-Coder-0.5B-Instruct, F3+F4 clean | `models/sft/sft_f3f4_instruct/` | `Qwen/Qwen2.5-Coder-0.5B-Instruct` | 491,004 records; step 15,033; reported pass@1 5.7%, pass@5 11.5% |
+| S4: Qwen2.5-Coder-0.5B-Instruct, F4 clean | `models/sft/sft_f4_instruct/` | `Qwen/Qwen2.5-Coder-0.5B-Instruct` | 17,539 records; step 484; reported pass@1 0.9%, pass@5 2.5% |
 
-All five directories are standard Hugging Face checkpoints. The 7B selection is the
+All nine directories are standard Hugging Face checkpoints. The 7B selection is the
 completed full-parameter continued-pre-training checkpoint, not the separate historical
 7B LoRA instruction-tuning experiment.
 
+The S1--S4 identifiers, exact source checkpoints, and evaluation results are catalogued
+in `models/sft/README.md`. In particular, S3 used 491,004 records (473,465 F3 records plus
+17,539 F4 records); this corrects the approximate 150K count in an earlier historical
+report. The checkpoint arguments, launch record, and measured input files support the
+491,004-record count.
+
 Model weights, tokenisers, and available effective-argument records are included.
 Optimizer, scheduler, and random-number-generator states are excluded because this is a
 model and training-recipe delivery, not a mid-run resume package.
@@ -34,6 +45,10 @@ and complete training log are in `evidence/training_runs/qwen25_coder_7b_cpt/`.
 May 2026 runs did not capture run-specific package snapshots; this limitation is documented
 in `environment/KNOWN_LIMITATIONS.md`.
 
+This S1--S4 increment contains the selected model checkpoints and their provenance
+catalogue. Additional S2--S4 datasets and variant-specific launch scripts remain in the
+local candidate staging area and are not part of this upload.
+
 Historical scripts intentionally retain the absolute paths, GPU assignments, Docker image,
 ports, and cache locations used at training time. Replace these machine-specific paths
 before rerunning.

SHA256SUMS

@@ -1,5 +1,5 @@
-ea9a02cac26ee3ce1a37db781f4bbf469f50c5917b26807408a0a9f7deaad9d0  MANIFEST.json
-940c1e59ae7a979993baae5210e23c46be80066d8d3bd50b906204633aba6926  README.md
+8834312aee2fc8ab4ab0c446a582ed31b465f1ffe99a03a01f6c541dfd5a2e7b  MANIFEST.json
+1158cfe52c727407545277df8e053a88b4ff634db6975f2d947c0fc9a80548e2  README.md
 c3cf181018c4b93052e1ec31fed8db92a4f1b5a6926feca192ad860a8f9319df  THIRD_PARTY_NOTICES.md
 570dfb0cd8b5991714d7c52b2d8e65ebd22a874e9d85877ab8ad22162f778ed1  code/f3_filtering/build_F3_v3.py
 5107add2818270eb0309219d5b73aedc43153a1da97165750b22dae80e7f047e  code/f3_filtering/build_F3_v4.py
@@ -65,7 +65,7 @@ e582cefbfb377aee6f3cc0c23f8159d5a77b14de27cf9fb2c4722078b905459a  data/pretrain/
 d506d7cb29a41e4f29d4953eb57e9df0d827668afdbb1e2cb42457eb2cec4ccf  environment/KNOWN_LIMITATIONS.md
 36b8a5aa5b20f8bb6b671cedbc0cf9f7a7246535e9e8494e068ba825f1b03003  evidence/eval/f3_refined_9p1_summary.json
 2da8bebeea85e5185a8560f21fc73faef47aedb5c5dde9783887f27e1331658c  evidence/instruction_recipe_results.md
-06dcd8ad990e189445f3d964c2defedabd2351a6b617314ca4b87753cc29fe9c  evidence/smoke_test_results.json
+4d6943fd6ac7a96437da19b5c4b4ddd83746f78ee6547f7dca690391be7804fe  evidence/smoke_test_results.json
 09ef906e1e4c47e55104fc0b3b715c7f694f105b8bd2ae20fa5fd9eeed4c5433  evidence/training_runs/TRAINING_RUN_SUMMARY.json
 e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855  evidence/training_runs/codegpt_multilingual_5epoch/DONE_00222615
 4f578b1d56f98afa0922ccdf43da84eb0165b7f68a522b110470031c03a72f56  evidence/training_runs/codegpt_multilingual_5epoch/build.log
@@ -120,6 +120,23 @@ c119b284587312405231891869c326215379ec62890899a0b6af324204f8d290  models/pretrai
 3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8  models/pretrained/stage1_cpt/tokenizer.json
 35c37fe281bb6e5ed2a6699349c8dc7191a79c4c829abbffab33133908f11daa  models/pretrained/stage1_cpt/tokenizer_config.json
 5fb959d771564eae82f9187cd41e4da522d985d8585158898d3a7525c4b292ca  models/pretrained/stage1_cpt/trainer_state.json
+cc0ab6ba18740855a558b91c686d7514c83cb2fe1e9a7f0dbfa2f398ef69a72d  models/sft/README.md
+77cbd14b2b1a955021b09903403339bddee49a5fe5c19d1149e984b1fdb213fd  models/sft/sft_f3_clean_instruct/args.json
+cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f  models/sft/sft_f3_clean_instruct/chat_template.jinja
+2a0bbab03998171855260860fbab81152ec51e67c7d69b85f32c7f624341fb0e  models/sft/sft_f3_clean_instruct/config.json
+87c51d61bd70a204993da36384296ccfeb56d46ee2cc7247929f917d0d8118f7  models/sft/sft_f3_clean_instruct/generation_config.json
+b0c56726f984cdb0ff6269ea300651a2a3e73c2827d07b36eccda22e2c6593eb  models/sft/sft_f3_clean_instruct/model.safetensors
+3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8  models/sft/sft_f3_clean_instruct/tokenizer.json
+4d8ad3fe9729e37338fae50bac94ff03cb915071eb886b937077114e874dfa06  models/sft/sft_f3_clean_instruct/tokenizer_config.json
+2e305da69a4ebf87c10d3945b684cfcd1f18fbfee04fa32ce92ce0a6b60b2354  models/sft/sft_f3_clean_instruct/trainer_state.json
+25791c8c6904fb5ad1459161de30caff274709dad969e17f5572fcda45fd81b1  models/sft/sft_f3_hq_instruct/args.json
+cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f  models/sft/sft_f3_hq_instruct/chat_template.jinja
+2a0bbab03998171855260860fbab81152ec51e67c7d69b85f32c7f624341fb0e  models/sft/sft_f3_hq_instruct/config.json
+87c51d61bd70a204993da36384296ccfeb56d46ee2cc7247929f917d0d8118f7  models/sft/sft_f3_hq_instruct/generation_config.json
+ac9c2e900b1cb9931417a77be5d8dc989d1d83222bcc359984690527ce21b621  models/sft/sft_f3_hq_instruct/model.safetensors
+3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8  models/sft/sft_f3_hq_instruct/tokenizer.json
+4d8ad3fe9729e37338fae50bac94ff03cb915071eb886b937077114e874dfa06  models/sft/sft_f3_hq_instruct/tokenizer_config.json
+2af634bb210f1a7d2ff5228364967f16a044416847d29c8b77be0221e5752d58  models/sft/sft_f3_hq_instruct/trainer_state.json
 4a04289e2de79f87bed89881e01536369dd8c34c1b01a4806fd0aa08ccfc8293  models/sft/sft_f3_refined_instruct/args.json
 cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f  models/sft/sft_f3_refined_instruct/chat_template.jinja
 2a0bbab03998171855260860fbab81152ec51e67c7d69b85f32c7f624341fb0e  models/sft/sft_f3_refined_instruct/config.json
@@ -128,8 +145,24 @@ cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f  models/sft/sft
 3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8  models/sft/sft_f3_refined_instruct/tokenizer.json
 4d8ad3fe9729e37338fae50bac94ff03cb915071eb886b937077114e874dfa06  models/sft/sft_f3_refined_instruct/tokenizer_config.json
 7259447195d9af3cad318bed168eb341c6ac2d171ccaddab7e195e173d033642  models/sft/sft_f3_refined_instruct/trainer_state.json
-6a0aa61077ee762b0a78ff8c5fc3c273f90ff61a96b3f3e4fa507695f28057c8  tools/build_manifest.py
-7d67ef5a41617dbf0e902c6c6d7b129c287cab50f4f08b150757dc396edf04cf  tools/smoke_test.py
+cbfd1a1eadf3449c55fd8d7c4bc87941ba158f7234efaabac72bd970b520686c  models/sft/sft_f3f4_instruct/args.json
+cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f  models/sft/sft_f3f4_instruct/chat_template.jinja
+2a0bbab03998171855260860fbab81152ec51e67c7d69b85f32c7f624341fb0e  models/sft/sft_f3f4_instruct/config.json
+87c51d61bd70a204993da36384296ccfeb56d46ee2cc7247929f917d0d8118f7  models/sft/sft_f3f4_instruct/generation_config.json
+3fc57c00fa0e875ea148c69d11a87a1af3f9b4b6571f2fdbeeaa17ba4220f373  models/sft/sft_f3f4_instruct/model.safetensors
+3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8  models/sft/sft_f3f4_instruct/tokenizer.json
+4d8ad3fe9729e37338fae50bac94ff03cb915071eb886b937077114e874dfa06  models/sft/sft_f3f4_instruct/tokenizer_config.json
+b95c19cafed50b446810a01b544e4a9e7643110e538b42e306fce3c3daf8d9a4  models/sft/sft_f3f4_instruct/trainer_state.json
+6955cf0f31569ffa12ad689770e64cbd5901eb40ad49cc34acc4eafb3b26a48c  models/sft/sft_f4_instruct/args.json
+cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f  models/sft/sft_f4_instruct/chat_template.jinja
+2a0bbab03998171855260860fbab81152ec51e67c7d69b85f32c7f624341fb0e  models/sft/sft_f4_instruct/config.json
+87c51d61bd70a204993da36384296ccfeb56d46ee2cc7247929f917d0d8118f7  models/sft/sft_f4_instruct/generation_config.json
+d9a6cc732c33b8ee955f2766c7be98a91272603d1be38f51569bf13e1ac292b1  models/sft/sft_f4_instruct/model.safetensors
+3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8  models/sft/sft_f4_instruct/tokenizer.json
+4d8ad3fe9729e37338fae50bac94ff03cb915071eb886b937077114e874dfa06  models/sft/sft_f4_instruct/tokenizer_config.json
+606a03e67a9539fd2ba2232dc6a76e4a397623e124fb9404005eb08e0dddb02d  models/sft/sft_f4_instruct/trainer_state.json
+099962f86157d5684e1d18a463c6a691cb5a3f0d8b8845688e23d243fad0352f  tools/build_manifest.py
+7f2dc8c83cb06184eea06972a031840de2d256bf88a7242b1204408a8c49039d  tools/smoke_test.py
 bc17ddd1ee0d2a78dfa9b512a2ac44a47681c536f5be33a412cfedd3f2971ec0  training_scripts/README.md
 83c8f4b93910e97dc1f53c2fd961c89b9770193d7b2e4922924da077b38a3168  training_scripts/pretrain/codegpt_multilingual_5epoch.sh
 f49b18ea579b871f4a08b1c7d30efcea595abff4734607a3b82e1d3485ffd297  training_scripts/pretrain/gpt2_124m_code_5epoch.sh

evidence/smoke_test_results.json

@@ -97,6 +97,78 @@
       ],
       "finite": true,
       "top1_zscore": 12.88
+    },
+    {
+      "model": "sft/sft_f3_clean_instruct",
+      "path": "models/sft/sft_f3_clean_instruct",
+      "ok": true,
+      "architecture": "Qwen2ForCausalLM",
+      "hidden_size": 896,
+      "layers": 24,
+      "vocab_size": 151936,
+      "tokenizer": "Qwen2Tokenizer",
+      "parameters_million": 494.0,
+      "logits_shape": [
+        1,
+        14,
+        151936
+      ],
+      "finite": true,
+      "top1_zscore": 12.96
+    },
+    {
+      "model": "sft/sft_f3_hq_instruct",
+      "path": "models/sft/sft_f3_hq_instruct",
+      "ok": true,
+      "architecture": "Qwen2ForCausalLM",
+      "hidden_size": 896,
+      "layers": 24,
+      "vocab_size": 151936,
+      "tokenizer": "Qwen2Tokenizer",
+      "parameters_million": 494.0,
+      "logits_shape": [
+        1,
+        14,
+        151936
+      ],
+      "finite": true,
+      "top1_zscore": 13.01
+    },
+    {
+      "model": "sft/sft_f3f4_instruct",
+      "path": "models/sft/sft_f3f4_instruct",
+      "ok": true,
+      "architecture": "Qwen2ForCausalLM",
+      "hidden_size": 896,
+      "layers": 24,
+      "vocab_size": 151936,
+      "tokenizer": "Qwen2Tokenizer",
+      "parameters_million": 494.0,
+      "logits_shape": [
+        1,
+        14,
+        151936
+      ],
+      "finite": true,
+      "top1_zscore": 12.88
+    },
+    {
+      "model": "sft/sft_f4_instruct",
+      "path": "models/sft/sft_f4_instruct",
+      "ok": true,
+      "architecture": "Qwen2ForCausalLM",
+      "hidden_size": 896,
+      "layers": 24,
+      "vocab_size": 151936,
+      "tokenizer": "Qwen2Tokenizer",
+      "parameters_million": 494.0,
+      "logits_shape": [
+        1,
+        14,
+        151936
+      ],
+      "finite": true,
+      "top1_zscore": 12.5
     }
   ]
 }

models/sft/README.md

@@ -0,0 +1,28 @@
+# Selected SFT Checkpoints
+
+This catalogue records the four SFT variants explicitly selected for the Milestone 2
+model-only increment. Reported pass rates are historical Runnable alignment results,
+not results on the later repository-level Core/Hold benchmark.
+
+| ID | Delivered directory | Initial model | Training data | Final checkpoint | Reported pass@1 | Reported pass@5 |
+|---|---|---|---:|---:|---:|---:|
+| S1 | `sft_f3_clean_instruct/` | `Qwen/Qwen2.5-Coder-0.5B-Instruct` | F3 clean, 473,465 records | step 14,550, epoch 1 | 9.1% | 13.0% |
+| S2 | `sft_f3_hq_instruct/` | `Qwen/Qwen2.5-Coder-0.5B-Instruct` | F3 high-quality v4, 134,775 records | step 4,212, epoch 1 | 8.3% | 13.0% |
+| S3 | `sft_f3f4_instruct/` | `Qwen/Qwen2.5-Coder-0.5B-Instruct` | F3 clean + F4 clean, 491,004 records | step 15,033, epoch 1 | 5.7% | 11.5% |
+| S4 | `sft_f4_instruct/` | `Qwen/Qwen2.5-Coder-0.5B-Instruct` | F4 clean, 17,539 records | step 484, epoch 1 | 0.9% | 2.5% |
+
+## Authoritative local sources
+
+| ID | Source checkpoint |
+|---|---|
+| S1 | `/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_clean_1epoch_1gpu/v1-20260505-184601/checkpoint-14550` |
+| S2 | `/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v4_1epoch_1gpu/v0-20260519-205734/checkpoint-4212` |
+| S3 | `/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3F4_clean_1epoch_1gpu/v0-20260506-103602/checkpoint-15033` |
+| S4 | `/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F4_clean_1epoch_1gpu/v3-20260505-141059/checkpoint-484` |
+
+S3 contains 491,004 examples: 473,465 F3 records plus 17,539 F4 records. This exact
+count supersedes the approximate 150K value in an earlier historical report.
+
+Each directory contains a complete Hugging Face inference checkpoint: configuration,
+tokeniser, chat template, generation configuration, safetensors weights, effective
+arguments, and trainer state. Optimiser and scheduler states are intentionally excluded.

models/sft/sft_f3_clean_instruct/args.json

@@ -0,0 +1,344 @@
+{
+  "output_dir": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_clean_1epoch_1gpu/v1-20260505-184601",
+  "per_device_train_batch_size": 4,
+  "num_train_epochs": 1.0,
+  "max_steps": -1,
+  "learning_rate": 2e-05,
+  "lr_scheduler_type": "cosine",
+  "lr_scheduler_kwargs": null,
+  "warmup_steps": 0,
+  "optim": "adamw_torch_fused",
+  "optim_args": null,
+  "weight_decay": 0.1,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.95,
+  "adam_epsilon": 1e-08,
+  "optim_target_modules": null,
+  "gradient_accumulation_steps": 8,
+  "average_tokens_across_devices": true,
+  "max_grad_norm": 1.0,
+  "label_smoothing_factor": 0.0,
+  "bf16": true,
+  "fp16": false,
+  "bf16_full_eval": false,
+  "fp16_full_eval": false,
+  "tf32": null,
+  "gradient_checkpointing": true,
+  "gradient_checkpointing_kwargs": null,
+  "torch_compile": false,
+  "torch_compile_backend": null,
+  "torch_compile_mode": null,
+  "use_liger_kernel": false,
+  "liger_kernel_config": null,
+  "use_cache": false,
+  "neftune_noise_alpha": null,
+  "torch_empty_cache_steps": null,
+  "auto_find_batch_size": false,
+  "logging_strategy": "steps",
+  "logging_steps": 10,
+  "logging_first_step": true,
+  "log_on_each_node": true,
+  "logging_nan_inf_filter": true,
+  "include_num_input_tokens_seen": false,
+  "log_level": "passive",
+  "log_level_replica": "warning",
+  "disable_tqdm": null,
+  "report_to": [
+    "tensorboard"
+  ],
+  "run_name": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_clean_1epoch_1gpu/v1-20260505-184601",
+  "project": "huggingface",
+  "trackio_space_id": null,
+  "trackio_bucket_id": null,
+  "trackio_static_space_id": null,
+  "eval_strategy": "no",
+  "eval_steps": 500.0,
+  "eval_delay": 0,
+  "per_device_eval_batch_size": 1,
+  "prediction_loss_only": false,
+  "eval_on_start": false,
+  "eval_do_concat_batches": true,
+  "eval_use_gather_object": false,
+  "eval_accumulation_steps": null,
+  "include_for_metrics": [],
+  "batch_eval_metrics": false,
+  "save_only_model": false,
+  "save_strategy": "steps",
+  "save_steps": 500.0,
+  "save_on_each_node": false,
+  "save_total_limit": 5,
+  "enable_jit_checkpoint": false,
+  "push_to_hub": false,
+  "hub_token": null,
+  "hub_private_repo": null,
+  "hub_model_id": null,
+  "hub_strategy": "every_save",
+  "hub_always_push": false,
+  "hub_revision": null,
+  "load_best_model_at_end": false,
+  "metric_for_best_model": "loss",
+  "greater_is_better": false,
+  "ignore_data_skip": false,
+  "restore_callback_states_from_checkpoint": false,
+  "full_determinism": false,
+  "seed": 42,
+  "data_seed": 42,
+  "use_cpu": false,
+  "accelerator_config": {
+    "dispatch_batches": false
+  },
+  "parallelism_config": null,
+  "dataloader_drop_last": false,
+  "dataloader_num_workers": 8,
+  "dataloader_pin_memory": true,
+  "dataloader_persistent_workers": false,
+  "dataloader_prefetch_factor": null,
+  "remove_unused_columns": true,
+  "label_names": null,
+  "train_sampling_strategy": "random",
+  "length_column_name": "length",
+  "ddp_find_unused_parameters": null,
+  "ddp_bucket_cap_mb": null,
+  "ddp_broadcast_buffers": null,
+  "ddp_static_graph": null,
+  "ddp_backend": null,
+  "ddp_timeout": 18000000,
+  "fsdp": [],
+  "fsdp_config": null,
+  "deepspeed": null,
+  "debug": null,
+  "skip_memory_metrics": true,
+  "do_train": false,
+  "do_eval": false,
+  "do_predict": false,
+  "resume_from_checkpoint": null,
+  "warmup_ratio": 0.03,
+  "logging_dir": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_clean_1epoch_1gpu/v1-20260505-184601/runs",
+  "local_rank": 0,
+  "sortish_sampler": false,
+  "predict_with_generate": false,
+  "generation_max_length": null,
+  "generation_num_beams": null,
+  "generation_config": null,
+  "tuner_backend": "peft",
+  "vit_gradient_checkpointing": false,
+  "router_aux_loss_coef": 0.0,
+  "enable_dft_loss": false,
+  "enable_channel_loss": false,
+  "safe_serialization": true,
+  "max_shard_size": "5GB",
+  "check_model": true,
+  "acc_strategy": "token",
+  "train_dataloader_shuffle": true,
+  "group_by_length": false,
+  "max_epochs": null,
+  "aligner_lr": null,
+  "vit_lr": null,
+  "use_logits_to_keep": null,
+  "ds3_gather_for_generation": true,
+  "resume_only_model": false,
+  "optimizer": null,
+  "loss_type": null,
+  "eval_metric": null,
+  "callbacks": [],
+  "early_stop_interval": null,
+  "eval_use_evalscope": false,
+  "eval_dataset": [],
+  "eval_dataset_args": null,
+  "eval_limit": null,
+  "eval_generation_config": null,
+  "extra_eval_args": null,
+  "tuner_type": "full",
+  "use_galore": false,
+  "galore_target_modules": null,
+  "galore_rank": 128,
+  "galore_update_proj_gap": 50,
+  "galore_scale": 1.0,
+  "galore_proj_type": "std",
+  "galore_optim_per_parameter": false,
+  "galore_with_embedding": false,
+  "galore_quantization": false,
+  "galore_proj_quant": false,
+  "galore_proj_bits": 4,
+  "galore_proj_group_size": 256,
+  "galore_cos_threshold": 0.4,
+  "galore_gamma_proj": 2,
+  "galore_queue_size": 5,
+  "lisa_activated_layers": 0,
+  "lisa_step_interval": 20,
+  "use_flash_ckpt": false,
+  "use_ray": false,
+  "ray_exp_name": null,
+  "device_groups": null,
+  "model": "Qwen/Qwen2.5-Coder-0.5B-Instruct",
+  "model_type": "qwen2",
+  "model_revision": null,
+  "task_type": "causal_lm",
+  "torch_dtype": "bfloat16",
+  "attn_impl": "flash_attn",
+  "experts_impl": null,
+  "new_special_tokens": [],
+  "num_labels": null,
+  "problem_type": null,
+  "rope_scaling": null,
+  "device_map": null,
+  "max_memory": {},
+  "max_model_len": null,
+  "local_repo_path": null,
+  "init_strategy": null,
+  "template": "qwen2_5",
+  "system": null,
+  "max_length": 4096,
+  "truncation_strategy": "delete",
+  "max_pixels": null,
+  "agent_template": null,
+  "norm_bbox": null,
+  "use_chat_template": true,
+  "padding_side": "right",
+  "padding_free": false,
+  "loss_scale": "default",
+  "sequence_parallel_size": 1,
+  "template_backend": "swift",
+  "response_prefix": null,
+  "enable_thinking": null,
+  "add_non_thinking_prefix": true,
+  "dataset": [
+    "/raid/data/weifeng/Datasets/Instruction-turning/scicode_sft_swift_F3_clean.jsonl"
+  ],
+  "val_dataset": [],
+  "cached_dataset": [],
+  "cached_val_dataset": [],
+  "split_dataset_ratio": 0.0,
+  "dataset_num_proc": 8,
+  "load_from_cache_file": false,
+  "dataset_shuffle": true,
+  "val_dataset_shuffle": false,
+  "streaming": false,
+  "interleave_prob": null,
+  "stopping_strategy": "first_exhausted",
+  "shuffle_buffer_size": 1000,
+  "download_mode": "reuse_dataset_if_exists",
+  "columns": {},
+  "strict": false,
+  "model_name": null,
+  "model_author": null,
+  "custom_dataset_info": [],
+  "quant_method": null,
+  "quant_bits": null,
+  "hqq_axis": null,
+  "bnb_4bit_compute_dtype": "bfloat16",
+  "bnb_4bit_quant_type": "nf4",
+  "bnb_4bit_use_double_quant": true,
+  "bnb_4bit_quant_storage": null,
+  "max_new_tokens": 64,
+  "temperature": 0.0,
+  "top_k": null,
+  "top_p": null,
+  "repetition_penalty": null,
+  "num_beams": 1,
+  "stream": false,
+  "stop_words": [],
+  "logprobs": false,
+  "top_logprobs": null,
+  "structured_outputs_regex": null,
+  "adapters": [],
+  "external_plugins": [],
+  "custom_register_path": [],
+  "model_kwargs": {},
+  "load_args": false,
+  "load_data_args": false,
+  "packing": false,
+  "packing_length": null,
+  "packing_num_proc": 1,
+  "lazy_tokenize": false,
+  "use_hf": true,
+  "ignore_args_error": false,
+  "use_swift_lora": false,
+  "freeze_parameters": [],
+  "freeze_parameters_regex": null,
+  "freeze_parameters_ratio": 0.0,
+  "trainable_parameters": [],
+  "trainable_parameters_regex": null,
+  "freeze_llm": false,
+  "freeze_vit": true,
+  "freeze_aligner": true,
+  "target_modules": [
+    "all-linear"
+  ],
+  "target_regex": null,
+  "target_parameters": null,
+  "modules_to_save": [],
+  "lora_rank": 8,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_bias": "none",
+  "lora_dtype": null,
+  "lorap_lr_ratio": null,
+  "use_rslora": false,
+  "use_dora": false,
+  "lora_ga_batch_size": 2,
+  "lora_ga_iters": 2,
+  "lora_ga_max_length": 1024,
+  "lora_ga_direction": "ArB2r",
+  "lora_ga_scale": "stable",
+  "lora_ga_stable_gamma": 16,
+  "init_weights": true,
+  "fourier_n_frequency": 2000,
+  "fourier_scaling": 300.0,
+  "boft_block_size": 4,
+  "boft_block_num": 0,
+  "boft_n_butterfly_factor": 1,
+  "boft_dropout": 0.0,
+  "vera_rank": 256,
+  "vera_projection_prng_key": 0,
+  "vera_dropout": 0.0,
+  "vera_d_initial": 0.1,
+  "adapter_act": "gelu",
+  "adapter_length": 128,
+  "adalora_target_r": 8,
+  "adalora_init_r": 12,
+  "adalora_tinit": 0,
+  "adalora_tfinal": 0,
+  "adalora_deltaT": 1,
+  "adalora_beta1": 0.85,
+  "adalora_beta2": 0.85,
+  "adalora_orth_reg_weight": 0.5,
+  "llamapro_num_new_blocks": 4,
+  "llamapro_num_groups": null,
+  "reft_layer_key": null,
+  "reft_layers": null,
+  "reft_rank": 4,
+  "reft_intervention_type": "LoreftIntervention",
+  "reft_args": null,
+  "swanlab_token": null,
+  "swanlab_project": "ms-swift",
+  "swanlab_workspace": null,
+  "swanlab_exp_name": null,
+  "swanlab_notification_method": null,
+  "swanlab_webhook_url": null,
+  "swanlab_secret": null,
+  "swanlab_sender_email": null,
+  "swanlab_receiver_email": null,
+  "swanlab_smtp_server": null,
+  "swanlab_smtp_port": null,
+  "swanlab_email_language": "zh",
+  "swanlab_mode": "cloud",
+  "add_version": true,
+  "create_checkpoint_symlink": false,
+  "zero_hpz_partition_size": null,
+  "deepspeed_autotp_size": null,
+  "swift_version": "4.1.3",
+  "ckpt_dir": null,
+  "rank": 0,
+  "global_world_size": 1,
+  "local_world_size": 1,
+  "model_suffix": "Qwen2.5-Coder-0.5B-Instruct",
+  "model_info": "ModelInfo(model_type='qwen2', model_dir='/home/weifeng/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-0.5B-Instruct/snapshots/ea3f2471cf1b1f0db85067f1ef93848e38e88c25', torch_dtype=torch.bfloat16, max_model_len=32768, quant_method=None, quant_bits=None, rope_scaling={'rope_theta': 1000000.0, 'rope_type': 'default'}, is_moe_model=False, is_multimodal=False, config=None, task_type='causal_lm', num_labels=None)",
+  "model_meta": "ModelMeta(model_type='qwen2', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen1.5-0.5B-Chat', hf_model_id='Qwen/Qwen1.5-0.5B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-1.8B-Chat', hf_model_id='Qwen/Qwen1.5-1.8B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-4B-Chat', hf_model_id='Qwen/Qwen1.5-4B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-7B-Chat', hf_model_id='Qwen/Qwen1.5-7B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-14B-Chat', hf_model_id='Qwen/Qwen1.5-14B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-32B-Chat', hf_model_id='Qwen/Qwen1.5-32B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-72B-Chat', hf_model_id='Qwen/Qwen1.5-72B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-110B-Chat', hf_model_id='Qwen/Qwen1.5-110B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-0.5B', hf_model_id='Qwen/Qwen1.5-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-1.8B', hf_model_id='Qwen/Qwen1.5-1.8B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-4B', hf_model_id='Qwen/Qwen1.5-4B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-7B', hf_model_id='Qwen/Qwen1.5-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-14B', hf_model_id='Qwen/Qwen1.5-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-32B', hf_model_id='Qwen/Qwen1.5-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-72B', hf_model_id='Qwen/Qwen1.5-72B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-110B', hf_model_id='Qwen/Qwen1.5-110B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-4B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-4B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-7B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-7B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-14B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-14B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-32B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-32B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-72B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-72B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-110B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-110B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-4B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-4B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-7B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-7B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-14B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-14B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-72B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-72B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-0.5B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-0.5B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-1.8B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-1.8B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-4B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-4B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-7B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-7B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-14B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-14B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-32B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-32B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-72B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-72B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-110B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-110B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/CodeQwen1.5-7B', hf_model_id='Qwen/CodeQwen1.5-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/CodeQwen1.5-7B-Chat', hf_model_id='Qwen/CodeQwen1.5-7B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/CodeQwen1.5-7B-Chat-AWQ', hf_model_id='Qwen/CodeQwen1.5-7B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=['coding']), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2-0.5B-Instruct', hf_model_id='Qwen/Qwen2-0.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-1.5B-Instruct', hf_model_id='Qwen/Qwen2-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-7B-Instruct', hf_model_id='Qwen/Qwen2-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-72B-Instruct', hf_model_id='Qwen/Qwen2-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-0.5B', hf_model_id='Qwen/Qwen2-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-1.5B', hf_model_id='Qwen/Qwen2-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-7B', hf_model_id='Qwen/Qwen2-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-72B', hf_model_id='Qwen/Qwen2-72B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-0.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2-0.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-7B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2-7B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-72B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2-72B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-0.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2-0.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-1.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2-1.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-7B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2-7B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-72B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2-72B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-0.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2-0.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-1.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2-1.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-72B-Instruct-AWQ', hf_model_id='Qwen/Qwen2-72B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2-Math-1.5B-Instruct', hf_model_id='Qwen/Qwen2-Math-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-Math-7B-Instruct', hf_model_id='Qwen/Qwen2-Math-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-Math-72B-Instruct', hf_model_id='Qwen/Qwen2-Math-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-Math-1.5B', hf_model_id='Qwen/Qwen2-Math-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-Math-7B', hf_model_id='Qwen/Qwen2-Math-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-Math-72B', hf_model_id='Qwen/Qwen2-Math-72B', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=['math']), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-1M', hf_model_id='Qwen/Qwen2.5-7B-Instruct-1M', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-1M', hf_model_id='Qwen/Qwen2.5-14B-Instruct-1M', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='PowerInfer/SmallThinker-3B-Preview', hf_model_id='PowerInfer/SmallThinker-3B-Preview', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct', hf_model_id='Qwen/Qwen2.5-3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct', hf_model_id='Qwen/Qwen2.5-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct', hf_model_id='Qwen/Qwen2.5-14B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct', hf_model_id='Qwen/Qwen2.5-32B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct', hf_model_id='Qwen/Qwen2.5-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B', hf_model_id='Qwen/Qwen2.5-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B', hf_model_id='Qwen/Qwen2.5-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B', hf_model_id='Qwen/Qwen2.5-3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B', hf_model_id='Qwen/Qwen2.5-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B', hf_model_id='Qwen/Qwen2.5-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B', hf_model_id='Qwen/Qwen2.5-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B', hf_model_id='Qwen/Qwen2.5-72B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-3B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-14B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-32B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-72B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwen2_5', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B', hf_model_id='Qwen/Qwen2.5-Coder-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B', hf_model_id='Qwen/Qwen2.5-Coder-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B', hf_model_id='Qwen/Qwen2.5-Coder-3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B', hf_model_id='Qwen/Qwen2.5-Coder-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B', hf_model_id='Qwen/Qwen2.5-Coder-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B', hf_model_id='Qwen/Qwen2.5-Coder-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None)], template='qwen2_5', ignore_patterns=None, requires=None, tags=['coding']), ModelGroup(models=[Model(ms_model_id='moonshotai/Kimi-Dev-72B', hf_model_id='moonshotai/Kimi-Dev-72B', model_path=None, ms_revision=None, hf_revision=None)], template='qwen2_5', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-Math-1.5B-Instruct', hf_model_id='Qwen/Qwen2.5-Math-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Math-7B-Instruct', hf_model_id='Qwen/Qwen2.5-Math-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Math-72B-Instruct', hf_model_id='Qwen/Qwen2.5-Math-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Math-1.5B', hf_model_id='Qwen/Qwen2.5-Math-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Math-7B', hf_model_id='Qwen/Qwen2.5-Math-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Math-72B', hf_model_id='Qwen/Qwen2.5-Math-72B', model_path=None, ms_revision=None, hf_revision=None)], template='qwen2_5_math', ignore_patterns=None, requires=None, tags=['math']), ModelGroup(models=[Model(ms_model_id='AIDC-AI/Marco-o1', hf_model_id='AIDC-AI/Marco-o1', model_path=None, ms_revision=None, hf_revision=None)], template='marco_o1', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/QwQ-32B-Preview', hf_model_id='Qwen/QwQ-32B-Preview', model_path=None, ms_revision=None, hf_revision=None)], template='qwq_preview', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/QwQ-32B', hf_model_id='Qwen/QwQ-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/QwQ-32B-AWQ', hf_model_id='Qwen/QwQ-32B-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwq', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B', hf_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-7B', hf_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-14B', hf_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-32B', hf_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='iic/QwenLong-L1-32B', hf_model_id='Tongyi-Zhiwen/QwenLong-L1-32B', model_path=None, ms_revision=None, hf_revision=None)], template='deepseek_r1', ignore_patterns=None, requires=None, tags=[])], loader=<class 'swift.model.register.ModelLoader'>, template='qwen2_5', model_arch=ModelKeys(arch_name='llama', embedding='model.embed_tokens', module_list='model.layers', lm_head='lm_head', q_proj='model.layers.{}.self_attn.q_proj', k_proj='model.layers.{}.self_attn.k_proj', v_proj='model.layers.{}.self_attn.v_proj', o_proj='model.layers.{}.self_attn.o_proj', attention='model.layers.{}.self_attn', mlp='model.layers.{}.mlp', down_proj='model.layers.{}.mlp.down_proj', qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None), mcore_model_type=None, architectures=['Qwen2ForCausalLM'], additional_saved_files=[], torch_dtype=None, is_multimodal=False, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.37'], tags=['coding'])",
+  "model_dir": "/home/weifeng/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-0.5B-Instruct/snapshots/ea3f2471cf1b1f0db85067f1ef93848e38e88c25",
+  "template_meta": "Qwen2_5TemplateMeta(template_type='qwen2_5', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=<class 'swift.template.base.Template'>, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system='You are Qwen, created by Alibaba Cloud. You are a helpful assistant.', auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='', non_thinking_prefix='', history_thinking_prefix='')",
+  "_val_dataset_exists": false,
+  "hub": "<class 'swift.hub.hub.HFHub'>",
+  "evaluation_strategy": "steps",
+  "training_args": "Seq2SeqTrainingArguments(output_dir='/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_clean_1epoch_1gpu/v1-20260505-184601', per_device_train_batch_size=4, num_train_epochs=1.0, max_steps=-1, learning_rate=2e-05, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, lr_scheduler_kwargs=None, warmup_steps=0.03, optim=<OptimizerNames.ADAMW_TORCH_FUSED: 'adamw_torch_fused'>, optim_args=None, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, optim_target_modules=None, gradient_accumulation_steps=8, average_tokens_across_devices=None, max_grad_norm=1.0, label_smoothing_factor=0.0, bf16=True, fp16=False, bf16_full_eval=False, fp16_full_eval=False, tf32=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, use_liger_kernel=False, liger_kernel_config=None, use_cache=False, neftune_noise_alpha=None, torch_empty_cache_steps=None, auto_find_batch_size=False, logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_steps=10, logging_first_step=True, log_on_each_node=True, logging_nan_inf_filter=True, include_num_input_tokens_seen=None, log_level='passive', log_level_replica='warning', disable_tqdm=False, report_to=['tensorboard'], run_name='/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_clean_1epoch_1gpu/v1-20260505-184601', project='huggingface', trackio_space_id=None, trackio_bucket_id=None, trackio_static_space_id=None, eval_strategy=<IntervalStrategy.NO: 'no'>, eval_steps=500.0, eval_delay=0, per_device_eval_batch_size=1, prediction_loss_only=False, eval_on_start=False, eval_do_concat_batches=True, eval_use_gather_object=False, eval_accumulation_steps=None, include_for_metrics=[], batch_eval_metrics=False, save_only_model=False, save_strategy=<SaveStrategy.STEPS: 'steps'>, save_steps=500, save_on_each_node=False, save_total_limit=5, enable_jit_checkpoint=False, push_to_hub=False, hub_token=None, hub_private_repo=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_always_push=False, hub_revision=None, load_best_model_at_end=False, metric_for_best_model='loss', greater_is_better=False, ignore_data_skip=False, restore_callback_states_from_checkpoint=False, full_determinism=False, seed=42, data_seed=42, use_cpu=False, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), parallelism_config=None, dataloader_drop_last=False, dataloader_num_workers=8, dataloader_pin_memory=True, dataloader_persistent_workers=False, dataloader_prefetch_factor=2, remove_unused_columns=False, label_names=None, train_sampling_strategy='random', length_column_name='length', ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, ddp_static_graph=None, ddp_backend=None, ddp_timeout=18000000, fsdp=[], fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, deepspeed=None, debug=[], skip_memory_metrics=True, do_train=False, do_eval=False, do_predict=False, resume_from_checkpoint=None, warmup_ratio=0.03, logging_dir='/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_clean_1epoch_1gpu/v1-20260505-184601/runs', local_rank=0, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, tuner_backend='peft', vit_gradient_checkpointing=False, router_aux_loss_coef=0.0, enable_dft_loss=False, enable_channel_loss=False, safe_serialization=True, max_shard_size='5GB', check_model=True, acc_strategy='token', train_dataloader_shuffle=True, group_by_length=False, max_epochs=None, aligner_lr=None, vit_lr=None, use_logits_to_keep=None, ds3_gather_for_generation=True, resume_only_model=False, optimizer=None, loss_type=None, eval_metric=None, callbacks=[], early_stop_interval=None, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, extra_eval_args=None, tuner_type='full', use_galore=False, galore_target_modules=None, galore_rank=128, galore_update_proj_gap=50, galore_scale=1.0, galore_proj_type='std', galore_optim_per_parameter=False, galore_with_embedding=False, galore_quantization=False, galore_proj_quant=False, galore_proj_bits=4, galore_proj_group_size=256, galore_cos_threshold=0.4, galore_gamma_proj=2, galore_queue_size=5, lisa_activated_layers=0, lisa_step_interval=20, use_flash_ckpt=False)"
+}

models/sft/sft_f3_clean_instruct/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

models/sft/sft_f3_clean_instruct/config.json

@@ -0,0 +1,57 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 896,
+  "initializer_range": 0.02,
+  "intermediate_size": 4864,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 24,
+  "model_type": "qwen2",
+  "num_attention_heads": 14,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 2,
+  "pad_token_id": 151643,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 1000000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.6.2",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

models/sft/sft_f3_clean_instruct/generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.05,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "5.6.2"
+}

models/sft/sft_f3_clean_instruct/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0c56726f984cdb0ff6269ea300651a2a3e73c2827d07b36eccda22e2c6593eb
+size 988097824

models/sft/sft_f3_clean_instruct/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892

models/sft/sft_f3_clean_instruct/tokenizer_config.json

@@ -0,0 +1,30 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

models/sft/sft_f3_hq_instruct/args.json

@@ -0,0 +1,344 @@
+{
+  "output_dir": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v4_1epoch_1gpu/v0-20260519-205734",
+  "per_device_train_batch_size": 4,
+  "num_train_epochs": 1.0,
+  "max_steps": -1,
+  "learning_rate": 2e-05,
+  "lr_scheduler_type": "cosine",
+  "lr_scheduler_kwargs": null,
+  "warmup_steps": 0,
+  "optim": "adamw_torch_fused",
+  "optim_args": null,
+  "weight_decay": 0.1,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.95,
+  "adam_epsilon": 1e-08,
+  "optim_target_modules": null,
+  "gradient_accumulation_steps": 8,
+  "average_tokens_across_devices": true,
+  "max_grad_norm": 1.0,
+  "label_smoothing_factor": 0.0,
+  "bf16": true,
+  "fp16": false,
+  "bf16_full_eval": false,
+  "fp16_full_eval": false,
+  "tf32": null,
+  "gradient_checkpointing": true,
+  "gradient_checkpointing_kwargs": null,
+  "torch_compile": false,
+  "torch_compile_backend": null,
+  "torch_compile_mode": null,
+  "use_liger_kernel": false,
+  "liger_kernel_config": null,
+  "use_cache": false,
+  "neftune_noise_alpha": null,
+  "torch_empty_cache_steps": null,
+  "auto_find_batch_size": false,
+  "logging_strategy": "steps",
+  "logging_steps": 10,
+  "logging_first_step": true,
+  "log_on_each_node": true,
+  "logging_nan_inf_filter": true,
+  "include_num_input_tokens_seen": false,
+  "log_level": "passive",
+  "log_level_replica": "warning",
+  "disable_tqdm": null,
+  "report_to": [
+    "tensorboard"
+  ],
+  "run_name": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v4_1epoch_1gpu/v0-20260519-205734",
+  "project": "huggingface",
+  "trackio_space_id": null,
+  "trackio_bucket_id": null,
+  "trackio_static_space_id": null,
+  "eval_strategy": "no",
+  "eval_steps": 500.0,
+  "eval_delay": 0,
+  "per_device_eval_batch_size": 1,
+  "prediction_loss_only": false,
+  "eval_on_start": false,
+  "eval_do_concat_batches": true,
+  "eval_use_gather_object": false,
+  "eval_accumulation_steps": null,
+  "include_for_metrics": [],
+  "batch_eval_metrics": false,
+  "save_only_model": false,
+  "save_strategy": "steps",
+  "save_steps": 500.0,
+  "save_on_each_node": false,
+  "save_total_limit": 5,
+  "enable_jit_checkpoint": false,
+  "push_to_hub": false,
+  "hub_token": null,
+  "hub_private_repo": null,
+  "hub_model_id": null,
+  "hub_strategy": "every_save",
+  "hub_always_push": false,
+  "hub_revision": null,
+  "load_best_model_at_end": false,
+  "metric_for_best_model": "loss",
+  "greater_is_better": false,
+  "ignore_data_skip": false,
+  "restore_callback_states_from_checkpoint": false,
+  "full_determinism": false,
+  "seed": 42,
+  "data_seed": 42,
+  "use_cpu": false,
+  "accelerator_config": {
+    "dispatch_batches": false
+  },
+  "parallelism_config": null,
+  "dataloader_drop_last": false,
+  "dataloader_num_workers": 8,
+  "dataloader_pin_memory": true,
+  "dataloader_persistent_workers": false,
+  "dataloader_prefetch_factor": null,
+  "remove_unused_columns": true,
+  "label_names": null,
+  "train_sampling_strategy": "random",
+  "length_column_name": "length",
+  "ddp_find_unused_parameters": null,
+  "ddp_bucket_cap_mb": null,
+  "ddp_broadcast_buffers": null,
+  "ddp_static_graph": null,
+  "ddp_backend": null,
+  "ddp_timeout": 18000000,
+  "fsdp": [],
+  "fsdp_config": null,
+  "deepspeed": null,
+  "debug": null,
+  "skip_memory_metrics": true,
+  "do_train": false,
+  "do_eval": false,
+  "do_predict": false,
+  "resume_from_checkpoint": null,
+  "warmup_ratio": 0.03,
+  "logging_dir": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v4_1epoch_1gpu/v0-20260519-205734/runs",
+  "local_rank": 0,
+  "sortish_sampler": false,
+  "predict_with_generate": false,
+  "generation_max_length": null,
+  "generation_num_beams": null,
+  "generation_config": null,
+  "tuner_backend": "peft",
+  "vit_gradient_checkpointing": false,
+  "router_aux_loss_coef": 0.0,
+  "enable_dft_loss": false,
+  "enable_channel_loss": false,
+  "safe_serialization": true,
+  "max_shard_size": "5GB",
+  "check_model": true,
+  "acc_strategy": "token",
+  "train_dataloader_shuffle": true,
+  "group_by_length": false,
+  "max_epochs": null,
+  "aligner_lr": null,
+  "vit_lr": null,
+  "use_logits_to_keep": null,
+  "ds3_gather_for_generation": true,
+  "resume_only_model": false,
+  "optimizer": null,
+  "loss_type": null,
+  "eval_metric": null,
+  "callbacks": [],
+  "early_stop_interval": null,
+  "eval_use_evalscope": false,
+  "eval_dataset": [],
+  "eval_dataset_args": null,
+  "eval_limit": null,
+  "eval_generation_config": null,
+  "extra_eval_args": null,
+  "tuner_type": "full",
+  "use_galore": false,
+  "galore_target_modules": null,
+  "galore_rank": 128,
+  "galore_update_proj_gap": 50,
+  "galore_scale": 1.0,
+  "galore_proj_type": "std",
+  "galore_optim_per_parameter": false,
+  "galore_with_embedding": false,
+  "galore_quantization": false,
+  "galore_proj_quant": false,
+  "galore_proj_bits": 4,
+  "galore_proj_group_size": 256,
+  "galore_cos_threshold": 0.4,
+  "galore_gamma_proj": 2,
+  "galore_queue_size": 5,
+  "lisa_activated_layers": 0,
+  "lisa_step_interval": 20,
+  "use_flash_ckpt": false,
+  "use_ray": false,
+  "ray_exp_name": null,
+  "device_groups": null,
+  "model": "Qwen/Qwen2.5-Coder-0.5B-Instruct",
+  "model_type": "qwen2",
+  "model_revision": null,
+  "task_type": "causal_lm",
+  "torch_dtype": "bfloat16",
+  "attn_impl": "flash_attn",
+  "experts_impl": null,
+  "new_special_tokens": [],
+  "num_labels": null,
+  "problem_type": null,
+  "rope_scaling": null,
+  "device_map": null,
+  "max_memory": {},
+  "max_model_len": null,
+  "local_repo_path": null,
+  "init_strategy": null,
+  "template": "qwen2_5",
+  "system": null,
+  "max_length": 4096,
+  "truncation_strategy": "delete",
+  "max_pixels": null,
+  "agent_template": null,
+  "norm_bbox": null,
+  "use_chat_template": true,
+  "padding_side": "right",
+  "padding_free": false,
+  "loss_scale": "default",
+  "sequence_parallel_size": 1,
+  "template_backend": "swift",
+  "response_prefix": null,
+  "enable_thinking": null,
+  "add_non_thinking_prefix": true,
+  "dataset": [
+    "/raid/data/weifeng/Datasets/Instruction-turning/scicode_sft_swift_F3_clean_v4.jsonl"
+  ],
+  "val_dataset": [],
+  "cached_dataset": [],
+  "cached_val_dataset": [],
+  "split_dataset_ratio": 0.0,
+  "dataset_num_proc": 8,
+  "load_from_cache_file": false,
+  "dataset_shuffle": true,
+  "val_dataset_shuffle": false,
+  "streaming": false,
+  "interleave_prob": null,
+  "stopping_strategy": "first_exhausted",
+  "shuffle_buffer_size": 1000,
+  "download_mode": "reuse_dataset_if_exists",
+  "columns": {},
+  "strict": false,
+  "model_name": null,
+  "model_author": null,
+  "custom_dataset_info": [],
+  "quant_method": null,
+  "quant_bits": null,
+  "hqq_axis": null,
+  "bnb_4bit_compute_dtype": "bfloat16",
+  "bnb_4bit_quant_type": "nf4",
+  "bnb_4bit_use_double_quant": true,
+  "bnb_4bit_quant_storage": null,
+  "max_new_tokens": 64,
+  "temperature": 0.0,
+  "top_k": null,
+  "top_p": null,
+  "repetition_penalty": null,
+  "num_beams": 1,
+  "stream": false,
+  "stop_words": [],
+  "logprobs": false,
+  "top_logprobs": null,
+  "structured_outputs_regex": null,
+  "adapters": [],
+  "external_plugins": [],
+  "custom_register_path": [],
+  "model_kwargs": {},
+  "load_args": false,
+  "load_data_args": false,
+  "packing": false,
+  "packing_length": null,
+  "packing_num_proc": 1,
+  "lazy_tokenize": false,
+  "use_hf": true,
+  "ignore_args_error": false,
+  "use_swift_lora": false,
+  "freeze_parameters": [],
+  "freeze_parameters_regex": null,
+  "freeze_parameters_ratio": 0.0,
+  "trainable_parameters": [],
+  "trainable_parameters_regex": null,
+  "freeze_llm": false,
+  "freeze_vit": true,
+  "freeze_aligner": true,
+  "target_modules": [
+    "all-linear"
+  ],
+  "target_regex": null,
+  "target_parameters": null,
+  "modules_to_save": [],
+  "lora_rank": 8,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_bias": "none",
+  "lora_dtype": null,
+  "lorap_lr_ratio": null,
+  "use_rslora": false,
+  "use_dora": false,
+  "lora_ga_batch_size": 2,
+  "lora_ga_iters": 2,
+  "lora_ga_max_length": 1024,
+  "lora_ga_direction": "ArB2r",
+  "lora_ga_scale": "stable",
+  "lora_ga_stable_gamma": 16,
+  "init_weights": true,
+  "fourier_n_frequency": 2000,
+  "fourier_scaling": 300.0,
+  "boft_block_size": 4,
+  "boft_block_num": 0,
+  "boft_n_butterfly_factor": 1,
+  "boft_dropout": 0.0,
+  "vera_rank": 256,
+  "vera_projection_prng_key": 0,
+  "vera_dropout": 0.0,
+  "vera_d_initial": 0.1,
+  "adapter_act": "gelu",
+  "adapter_length": 128,
+  "adalora_target_r": 8,
+  "adalora_init_r": 12,
+  "adalora_tinit": 0,
+  "adalora_tfinal": 0,
+  "adalora_deltaT": 1,
+  "adalora_beta1": 0.85,
+  "adalora_beta2": 0.85,
+  "adalora_orth_reg_weight": 0.5,
+  "llamapro_num_new_blocks": 4,
+  "llamapro_num_groups": null,
+  "reft_layer_key": null,
+  "reft_layers": null,
+  "reft_rank": 4,
+  "reft_intervention_type": "LoreftIntervention",
+  "reft_args": null,
+  "swanlab_token": null,
+  "swanlab_project": "ms-swift",
+  "swanlab_workspace": null,
+  "swanlab_exp_name": null,
+  "swanlab_notification_method": null,
+  "swanlab_webhook_url": null,
+  "swanlab_secret": null,
+  "swanlab_sender_email": null,
+  "swanlab_receiver_email": null,
+  "swanlab_smtp_server": null,
+  "swanlab_smtp_port": null,
+  "swanlab_email_language": "zh",
+  "swanlab_mode": "cloud",
+  "add_version": true,
+  "create_checkpoint_symlink": false,
+  "zero_hpz_partition_size": null,
+  "deepspeed_autotp_size": null,
+  "swift_version": "4.1.3",
+  "ckpt_dir": null,
+  "rank": 0,
+  "global_world_size": 1,
+  "local_world_size": 1,
+  "model_suffix": "Qwen2.5-Coder-0.5B-Instruct",
+  "model_info": "ModelInfo(model_type='qwen2', model_dir='/home/weifeng/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-0.5B-Instruct/snapshots/ea3f2471cf1b1f0db85067f1ef93848e38e88c25', torch_dtype=torch.bfloat16, max_model_len=32768, quant_method=None, quant_bits=None, rope_scaling={'rope_theta': 1000000.0, 'rope_type': 'default'}, is_moe_model=False, is_multimodal=False, config=None, task_type='causal_lm', num_labels=None)",
+  "model_meta": "ModelMeta(model_type='qwen2', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen1.5-0.5B-Chat', hf_model_id='Qwen/Qwen1.5-0.5B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-1.8B-Chat', hf_model_id='Qwen/Qwen1.5-1.8B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-4B-Chat', hf_model_id='Qwen/Qwen1.5-4B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-7B-Chat', hf_model_id='Qwen/Qwen1.5-7B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-14B-Chat', hf_model_id='Qwen/Qwen1.5-14B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-32B-Chat', hf_model_id='Qwen/Qwen1.5-32B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-72B-Chat', hf_model_id='Qwen/Qwen1.5-72B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-110B-Chat', hf_model_id='Qwen/Qwen1.5-110B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-0.5B', hf_model_id='Qwen/Qwen1.5-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-1.8B', hf_model_id='Qwen/Qwen1.5-1.8B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-4B', hf_model_id='Qwen/Qwen1.5-4B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-7B', hf_model_id='Qwen/Qwen1.5-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-14B', hf_model_id='Qwen/Qwen1.5-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-32B', hf_model_id='Qwen/Qwen1.5-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-72B', hf_model_id='Qwen/Qwen1.5-72B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-110B', hf_model_id='Qwen/Qwen1.5-110B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-4B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-4B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-7B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-7B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-14B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-14B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-32B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-32B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-72B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-72B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-110B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-110B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-4B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-4B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-7B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-7B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-14B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-14B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-72B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-72B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-0.5B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-0.5B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-1.8B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-1.8B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-4B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-4B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-7B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-7B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-14B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-14B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-32B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-32B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-72B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-72B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-110B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-110B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/CodeQwen1.5-7B', hf_model_id='Qwen/CodeQwen1.5-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/CodeQwen1.5-7B-Chat', hf_model_id='Qwen/CodeQwen1.5-7B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/CodeQwen1.5-7B-Chat-AWQ', hf_model_id='Qwen/CodeQwen1.5-7B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=['coding']), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2-0.5B-Instruct', hf_model_id='Qwen/Qwen2-0.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-1.5B-Instruct', hf_model_id='Qwen/Qwen2-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-7B-Instruct', hf_model_id='Qwen/Qwen2-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-72B-Instruct', hf_model_id='Qwen/Qwen2-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-0.5B', hf_model_id='Qwen/Qwen2-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-1.5B', hf_model_id='Qwen/Qwen2-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-7B', hf_model_id='Qwen/Qwen2-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-72B', hf_model_id='Qwen/Qwen2-72B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-0.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2-0.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-7B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2-7B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-72B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2-72B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-0.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2-0.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-1.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2-1.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-7B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2-7B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-72B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2-72B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-0.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2-0.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-1.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2-1.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-72B-Instruct-AWQ', hf_model_id='Qwen/Qwen2-72B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2-Math-1.5B-Instruct', hf_model_id='Qwen/Qwen2-Math-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-Math-7B-Instruct', hf_model_id='Qwen/Qwen2-Math-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-Math-72B-Instruct', hf_model_id='Qwen/Qwen2-Math-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-Math-1.5B', hf_model_id='Qwen/Qwen2-Math-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-Math-7B', hf_model_id='Qwen/Qwen2-Math-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-Math-72B', hf_model_id='Qwen/Qwen2-Math-72B', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=['math']), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-1M', hf_model_id='Qwen/Qwen2.5-7B-Instruct-1M', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-1M', hf_model_id='Qwen/Qwen2.5-14B-Instruct-1M', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='PowerInfer/SmallThinker-3B-Preview', hf_model_id='PowerInfer/SmallThinker-3B-Preview', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct', hf_model_id='Qwen/Qwen2.5-3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct', hf_model_id='Qwen/Qwen2.5-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct', hf_model_id='Qwen/Qwen2.5-14B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct', hf_model_id='Qwen/Qwen2.5-32B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct', hf_model_id='Qwen/Qwen2.5-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B', hf_model_id='Qwen/Qwen2.5-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B', hf_model_id='Qwen/Qwen2.5-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B', hf_model_id='Qwen/Qwen2.5-3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B', hf_model_id='Qwen/Qwen2.5-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B', hf_model_id='Qwen/Qwen2.5-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B', hf_model_id='Qwen/Qwen2.5-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B', hf_model_id='Qwen/Qwen2.5-72B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-3B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-14B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-32B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-72B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwen2_5', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B', hf_model_id='Qwen/Qwen2.5-Coder-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B', hf_model_id='Qwen/Qwen2.5-Coder-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B', hf_model_id='Qwen/Qwen2.5-Coder-3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B', hf_model_id='Qwen/Qwen2.5-Coder-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B', hf_model_id='Qwen/Qwen2.5-Coder-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B', hf_model_id='Qwen/Qwen2.5-Coder-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None)], template='qwen2_5', ignore_patterns=None, requires=None, tags=['coding']), ModelGroup(models=[Model(ms_model_id='moonshotai/Kimi-Dev-72B', hf_model_id='moonshotai/Kimi-Dev-72B', model_path=None, ms_revision=None, hf_revision=None)], template='qwen2_5', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-Math-1.5B-Instruct', hf_model_id='Qwen/Qwen2.5-Math-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Math-7B-Instruct', hf_model_id='Qwen/Qwen2.5-Math-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Math-72B-Instruct', hf_model_id='Qwen/Qwen2.5-Math-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Math-1.5B', hf_model_id='Qwen/Qwen2.5-Math-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Math-7B', hf_model_id='Qwen/Qwen2.5-Math-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Math-72B', hf_model_id='Qwen/Qwen2.5-Math-72B', model_path=None, ms_revision=None, hf_revision=None)], template='qwen2_5_math', ignore_patterns=None, requires=None, tags=['math']), ModelGroup(models=[Model(ms_model_id='AIDC-AI/Marco-o1', hf_model_id='AIDC-AI/Marco-o1', model_path=None, ms_revision=None, hf_revision=None)], template='marco_o1', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/QwQ-32B-Preview', hf_model_id='Qwen/QwQ-32B-Preview', model_path=None, ms_revision=None, hf_revision=None)], template='qwq_preview', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/QwQ-32B', hf_model_id='Qwen/QwQ-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/QwQ-32B-AWQ', hf_model_id='Qwen/QwQ-32B-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwq', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B', hf_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-7B', hf_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-14B', hf_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-32B', hf_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='iic/QwenLong-L1-32B', hf_model_id='Tongyi-Zhiwen/QwenLong-L1-32B', model_path=None, ms_revision=None, hf_revision=None)], template='deepseek_r1', ignore_patterns=None, requires=None, tags=[])], loader=<class 'swift.model.register.ModelLoader'>, template='qwen2_5', model_arch=ModelKeys(arch_name='llama', embedding='model.embed_tokens', module_list='model.layers', lm_head='lm_head', q_proj='model.layers.{}.self_attn.q_proj', k_proj='model.layers.{}.self_attn.k_proj', v_proj='model.layers.{}.self_attn.v_proj', o_proj='model.layers.{}.self_attn.o_proj', attention='model.layers.{}.self_attn', mlp='model.layers.{}.mlp', down_proj='model.layers.{}.mlp.down_proj', qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None), mcore_model_type=None, architectures=['Qwen2ForCausalLM'], additional_saved_files=[], torch_dtype=None, is_multimodal=False, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.37'], tags=['coding'])",
+  "model_dir": "/home/weifeng/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-0.5B-Instruct/snapshots/ea3f2471cf1b1f0db85067f1ef93848e38e88c25",
+  "template_meta": "Qwen2_5TemplateMeta(template_type='qwen2_5', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=<class 'swift.template.base.Template'>, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system='You are Qwen, created by Alibaba Cloud. You are a helpful assistant.', auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='', non_thinking_prefix='', history_thinking_prefix='')",
+  "_val_dataset_exists": false,
+  "hub": "<class 'swift.hub.hub.HFHub'>",
+  "evaluation_strategy": "steps",
+  "training_args": "Seq2SeqTrainingArguments(output_dir='/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v4_1epoch_1gpu/v0-20260519-205734', per_device_train_batch_size=4, num_train_epochs=1.0, max_steps=-1, learning_rate=2e-05, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, lr_scheduler_kwargs=None, warmup_steps=0.03, optim=<OptimizerNames.ADAMW_TORCH_FUSED: 'adamw_torch_fused'>, optim_args=None, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, optim_target_modules=None, gradient_accumulation_steps=8, average_tokens_across_devices=None, max_grad_norm=1.0, label_smoothing_factor=0.0, bf16=True, fp16=False, bf16_full_eval=False, fp16_full_eval=False, tf32=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, use_liger_kernel=False, liger_kernel_config=None, use_cache=False, neftune_noise_alpha=None, torch_empty_cache_steps=None, auto_find_batch_size=False, logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_steps=10, logging_first_step=True, log_on_each_node=True, logging_nan_inf_filter=True, include_num_input_tokens_seen=None, log_level='passive', log_level_replica='warning', disable_tqdm=False, report_to=['tensorboard'], run_name='/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v4_1epoch_1gpu/v0-20260519-205734', project='huggingface', trackio_space_id=None, trackio_bucket_id=None, trackio_static_space_id=None, eval_strategy=<IntervalStrategy.NO: 'no'>, eval_steps=500.0, eval_delay=0, per_device_eval_batch_size=1, prediction_loss_only=False, eval_on_start=False, eval_do_concat_batches=True, eval_use_gather_object=False, eval_accumulation_steps=None, include_for_metrics=[], batch_eval_metrics=False, save_only_model=False, save_strategy=<SaveStrategy.STEPS: 'steps'>, save_steps=500, save_on_each_node=False, save_total_limit=5, enable_jit_checkpoint=False, push_to_hub=False, hub_token=None, hub_private_repo=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_always_push=False, hub_revision=None, load_best_model_at_end=False, metric_for_best_model='loss', greater_is_better=False, ignore_data_skip=False, restore_callback_states_from_checkpoint=False, full_determinism=False, seed=42, data_seed=42, use_cpu=False, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), parallelism_config=None, dataloader_drop_last=False, dataloader_num_workers=8, dataloader_pin_memory=True, dataloader_persistent_workers=False, dataloader_prefetch_factor=2, remove_unused_columns=False, label_names=None, train_sampling_strategy='random', length_column_name='length', ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, ddp_static_graph=None, ddp_backend=None, ddp_timeout=18000000, fsdp=[], fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, deepspeed=None, debug=[], skip_memory_metrics=True, do_train=False, do_eval=False, do_predict=False, resume_from_checkpoint=None, warmup_ratio=0.03, logging_dir='/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v4_1epoch_1gpu/v0-20260519-205734/runs', local_rank=0, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, tuner_backend='peft', vit_gradient_checkpointing=False, router_aux_loss_coef=0.0, enable_dft_loss=False, enable_channel_loss=False, safe_serialization=True, max_shard_size='5GB', check_model=True, acc_strategy='token', train_dataloader_shuffle=True, group_by_length=False, max_epochs=None, aligner_lr=None, vit_lr=None, use_logits_to_keep=None, ds3_gather_for_generation=True, resume_only_model=False, optimizer=None, loss_type=None, eval_metric=None, callbacks=[], early_stop_interval=None, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, extra_eval_args=None, tuner_type='full', use_galore=False, galore_target_modules=None, galore_rank=128, galore_update_proj_gap=50, galore_scale=1.0, galore_proj_type='std', galore_optim_per_parameter=False, galore_with_embedding=False, galore_quantization=False, galore_proj_quant=False, galore_proj_bits=4, galore_proj_group_size=256, galore_cos_threshold=0.4, galore_gamma_proj=2, galore_queue_size=5, lisa_activated_layers=0, lisa_step_interval=20, use_flash_ckpt=False)"
+}

models/sft/sft_f3_hq_instruct/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

models/sft/sft_f3_hq_instruct/config.json

@@ -0,0 +1,57 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 896,
+  "initializer_range": 0.02,
+  "intermediate_size": 4864,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 24,
+  "model_type": "qwen2",
+  "num_attention_heads": 14,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 2,
+  "pad_token_id": 151643,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 1000000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.6.2",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

models/sft/sft_f3_hq_instruct/generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.05,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "5.6.2"
+}

models/sft/sft_f3_hq_instruct/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac9c2e900b1cb9931417a77be5d8dc989d1d83222bcc359984690527ce21b621
+size 988097824

models/sft/sft_f3_hq_instruct/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892

models/sft/sft_f3_hq_instruct/tokenizer_config.json

@@ -0,0 +1,30 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

models/sft/sft_f3_hq_instruct/trainer_state.json

@@ -0,0 +1,3410 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500.0,
+  "global_step": 4212,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.00023744509082274725,
+      "grad_norm": 7.375,
+      "learning_rate": 1.5748031496062994e-07,
+      "loss": 0.9464934468269348,
+      "step": 1,
+      "token_acc": 0.7826174227217985
+    },
+    {
+      "epoch": 0.0023744509082274726,
+      "grad_norm": 7.3125,
+      "learning_rate": 1.5748031496062992e-06,
+      "loss": 1.0233561197916667,
+      "step": 10,
+      "token_acc": 0.7757428351372168
+    },
+    {
+      "epoch": 0.004748901816454945,
+      "grad_norm": 6.25,
+      "learning_rate": 3.1496062992125985e-06,
+      "loss": 1.081596565246582,
+      "step": 20,
+      "token_acc": 0.7621702306632909
+    },
+    {
+      "epoch": 0.007123352724682417,
+      "grad_norm": 5.28125,
+      "learning_rate": 4.724409448818898e-06,
+      "loss": 1.086085605621338,
+      "step": 30,
+      "token_acc": 0.7596227760827357
+    },
+    {
+      "epoch": 0.00949780363290989,
+      "grad_norm": 2.859375,
+      "learning_rate": 6.299212598425197e-06,
+      "loss": 0.978515338897705,
+      "step": 40,
+      "token_acc": 0.7724285926865754
+    },
+    {
+      "epoch": 0.011872254541137363,
+      "grad_norm": 2.421875,
+      "learning_rate": 7.874015748031496e-06,
+      "loss": 0.9195015907287598,
+      "step": 50,
+      "token_acc": 0.7831654958241626
+    },
+    {
+      "epoch": 0.014246705449364834,
+      "grad_norm": 2.1875,
+      "learning_rate": 9.448818897637797e-06,
+      "loss": 0.8691489219665527,
+      "step": 60,
+      "token_acc": 0.792770723715303
+    },
+    {
+      "epoch": 0.016621156357592308,
+      "grad_norm": 2.484375,
+      "learning_rate": 1.1023622047244095e-05,
+      "loss": 0.8688799858093261,
+      "step": 70,
+      "token_acc": 0.7934522469333467
+    },
+    {
+      "epoch": 0.01899560726581978,
+      "grad_norm": 2.3125,
+      "learning_rate": 1.2598425196850394e-05,
+      "loss": 0.8655830383300781,
+      "step": 80,
+      "token_acc": 0.7907126114628429
+    },
+    {
+      "epoch": 0.021370058174047253,
+      "grad_norm": 1.9296875,
+      "learning_rate": 1.4173228346456694e-05,
+      "loss": 0.8636247634887695,
+      "step": 90,
+      "token_acc": 0.7924484476217514
+    },
+    {
+      "epoch": 0.023744509082274726,
+      "grad_norm": 2.1875,
+      "learning_rate": 1.5748031496062993e-05,
+      "loss": 0.9061047554016113,
+      "step": 100,
+      "token_acc": 0.7866065175968601
+    },
+    {
+      "epoch": 0.026118959990502195,
+      "grad_norm": 2.0,
+      "learning_rate": 1.7322834645669295e-05,
+      "loss": 0.862424373626709,
+      "step": 110,
+      "token_acc": 0.7959050454552692
+    },
+    {
+      "epoch": 0.028493410898729667,
+      "grad_norm": 2.25,
+      "learning_rate": 1.8897637795275593e-05,
+      "loss": 0.8781260490417481,
+      "step": 120,
+      "token_acc": 0.7896513118398208
+    },
+    {
+      "epoch": 0.03086786180695714,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.999997338490965e-05,
+      "loss": 0.8858593940734864,
+      "step": 130,
+      "token_acc": 0.7885986095865349
+    },
+    {
+      "epoch": 0.033242312715184616,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.9999500231689075e-05,
+      "loss": 0.8478639602661133,
+      "step": 140,
+      "token_acc": 0.7956532665493904
+    },
+    {
+      "epoch": 0.03561676362341209,
+      "grad_norm": 1.9921875,
+      "learning_rate": 1.999843566422751e-05,
+      "loss": 0.9040656089782715,
+      "step": 150,
+      "token_acc": 0.7866907157774824
+    },
+    {
+      "epoch": 0.03799121453163956,
+      "grad_norm": 2.34375,
+      "learning_rate": 1.9996779745488144e-05,
+      "loss": 0.8792854309082031,
+      "step": 160,
+      "token_acc": 0.7898396576223589
+    },
+    {
+      "epoch": 0.040365665439867034,
+      "grad_norm": 1.875,
+      "learning_rate": 1.999453257340926e-05,
+      "loss": 0.8214083671569824,
+      "step": 170,
+      "token_acc": 0.8024454467675082
+    },
+    {
+      "epoch": 0.042740116348094506,
+      "grad_norm": 2.25,
+      "learning_rate": 1.9991694280898464e-05,
+      "loss": 0.8254429817199707,
+      "step": 180,
+      "token_acc": 0.8019397242705996
+    },
+    {
+      "epoch": 0.04511456725632198,
+      "grad_norm": 2.109375,
+      "learning_rate": 1.9988265035824802e-05,
+      "loss": 0.8791286468505859,
+      "step": 190,
+      "token_acc": 0.7902859240399559
+    },
+    {
+      "epoch": 0.04748901816454945,
+      "grad_norm": 2.1875,
+      "learning_rate": 1.9984245041008843e-05,
+      "loss": 0.9240189552307129,
+      "step": 200,
+      "token_acc": 0.7818896397578885
+    },
+    {
+      "epoch": 0.04986346907277692,
+      "grad_norm": 2.5625,
+      "learning_rate": 1.9979634534210697e-05,
+      "loss": 0.8700326919555664,
+      "step": 210,
+      "token_acc": 0.7885579382254574
+    },
+    {
+      "epoch": 0.05223791998100439,
+      "grad_norm": 2.421875,
+      "learning_rate": 1.9974433788115924e-05,
+      "loss": 0.8171942710876465,
+      "step": 220,
+      "token_acc": 0.8044196109734068
+    },
+    {
+      "epoch": 0.05461237088923186,
+      "grad_norm": 1.953125,
+      "learning_rate": 1.996864311031943e-05,
+      "loss": 0.8776941299438477,
+      "step": 230,
+      "token_acc": 0.7898050135060494
+    },
+    {
+      "epoch": 0.056986821797459335,
+      "grad_norm": 2.109375,
+      "learning_rate": 1.996226284330726e-05,
+      "loss": 0.8761281967163086,
+      "step": 240,
+      "token_acc": 0.7927178357974206
+    },
+    {
+      "epoch": 0.05936127270568681,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.995529336443635e-05,
+      "loss": 0.8758235931396484,
+      "step": 250,
+      "token_acc": 0.7891453878848836
+    },
+    {
+      "epoch": 0.06173572361391428,
+      "grad_norm": 2.5625,
+      "learning_rate": 1.994773508591221e-05,
+      "loss": 0.8513253211975098,
+      "step": 260,
+      "token_acc": 0.797296640642716
+    },
+    {
+      "epoch": 0.06411017452214175,
+      "grad_norm": 2.34375,
+      "learning_rate": 1.9939588454764534e-05,
+      "loss": 0.847078514099121,
+      "step": 270,
+      "token_acc": 0.7968817063022818
+    },
+    {
+      "epoch": 0.06648462543036923,
+      "grad_norm": 2.296875,
+      "learning_rate": 1.993085395282077e-05,
+      "loss": 0.8517152786254882,
+      "step": 280,
+      "token_acc": 0.793816550285949
+    },
+    {
+      "epoch": 0.0688590763385967,
+      "grad_norm": 2.390625,
+      "learning_rate": 1.992153209667762e-05,
+      "loss": 0.8805038452148437,
+      "step": 290,
+      "token_acc": 0.7906475237834565
+    },
+    {
+      "epoch": 0.07123352724682418,
+      "grad_norm": 2.203125,
+      "learning_rate": 1.991162343767049e-05,
+      "loss": 0.8262736320495605,
+      "step": 300,
+      "token_acc": 0.8024939727634065
+    },
+    {
+      "epoch": 0.07360797815505164,
+      "grad_norm": 2.265625,
+      "learning_rate": 1.9901128561840865e-05,
+      "loss": 0.8252796173095703,
+      "step": 310,
+      "token_acc": 0.8019994931354814
+    },
+    {
+      "epoch": 0.07598242906327912,
+      "grad_norm": 2.03125,
+      "learning_rate": 1.9890048089901674e-05,
+      "loss": 0.8695602416992188,
+      "step": 320,
+      "token_acc": 0.7938316003439919
+    },
+    {
+      "epoch": 0.07835687997150659,
+      "grad_norm": 2.125,
+      "learning_rate": 1.9878382677200558e-05,
+      "loss": 0.9021699905395508,
+      "step": 330,
+      "token_acc": 0.7856320980274956
+    },
+    {
+      "epoch": 0.08073133087973407,
+      "grad_norm": 2.296875,
+      "learning_rate": 1.9866133013681117e-05,
+      "loss": 0.8705177307128906,
+      "step": 340,
+      "token_acc": 0.7919138129509047
+    },
+    {
+      "epoch": 0.08310578178796153,
+      "grad_norm": 2.03125,
+      "learning_rate": 1.9853299823842106e-05,
+      "loss": 0.8325584411621094,
+      "step": 350,
+      "token_acc": 0.8009847792762801
+    },
+    {
+      "epoch": 0.08548023269618901,
+      "grad_norm": 2.046875,
+      "learning_rate": 1.983988386669458e-05,
+      "loss": 0.8228348731994629,
+      "step": 360,
+      "token_acc": 0.8021818397414994
+    },
+    {
+      "epoch": 0.08785468360441648,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.9825885935717007e-05,
+      "loss": 0.837125015258789,
+      "step": 370,
+      "token_acc": 0.7989180822160772
+    },
+    {
+      "epoch": 0.09022913451264396,
+      "grad_norm": 2.46875,
+      "learning_rate": 1.981130685880833e-05,
+      "loss": 0.8356122970581055,
+      "step": 380,
+      "token_acc": 0.8006993540899998
+    },
+    {
+      "epoch": 0.09260358542087142,
+      "grad_norm": 2.203125,
+      "learning_rate": 1.979614749823902e-05,
+      "loss": 0.8311405181884766,
+      "step": 390,
+      "token_acc": 0.7997111179252134
+    },
+    {
+      "epoch": 0.0949780363290989,
+      "grad_norm": 2.28125,
+      "learning_rate": 1.978040875060006e-05,
+      "loss": 0.8256620407104492,
+      "step": 400,
+      "token_acc": 0.8026782229738519
+    },
+    {
+      "epoch": 0.09735248723732637,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.976409154674991e-05,
+      "loss": 0.8367603302001954,
+      "step": 410,
+      "token_acc": 0.7980480045071543
+    },
+    {
+      "epoch": 0.09972693814555383,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.974719685175948e-05,
+      "loss": 0.8432640075683594,
+      "step": 420,
+      "token_acc": 0.7951476776078588
+    },
+    {
+      "epoch": 0.10210138905378131,
+      "grad_norm": 1.9609375,
+      "learning_rate": 1.972972566485503e-05,
+      "loss": 0.8305618286132812,
+      "step": 430,
+      "token_acc": 0.7980767752548158
+    },
+    {
+      "epoch": 0.10447583996200878,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.971167901935908e-05,
+      "loss": 0.8324492454528809,
+      "step": 440,
+      "token_acc": 0.798491060154947
+    },
+    {
+      "epoch": 0.10685029087023626,
+      "grad_norm": 2.0625,
+      "learning_rate": 1.9693057982629277e-05,
+      "loss": 0.811087703704834,
+      "step": 450,
+      "token_acc": 0.802839841302986
+    },
+    {
+      "epoch": 0.10922474177846372,
+      "grad_norm": 2.546875,
+      "learning_rate": 1.96738636559953e-05,
+      "loss": 0.8001727104187012,
+      "step": 460,
+      "token_acc": 0.80539893639436
+    },
+    {
+      "epoch": 0.1115991926866912,
+      "grad_norm": 2.25,
+      "learning_rate": 1.9654097174693702e-05,
+      "loss": 0.8267153739929199,
+      "step": 470,
+      "token_acc": 0.8016258799865906
+    },
+    {
+      "epoch": 0.11397364359491867,
+      "grad_norm": 2.125,
+      "learning_rate": 1.963375970780075e-05,
+      "loss": 0.8282063484191895,
+      "step": 480,
+      "token_acc": 0.8013324525505108
+    },
+    {
+      "epoch": 0.11634809450314615,
+      "grad_norm": 2.1875,
+      "learning_rate": 1.9612852458163325e-05,
+      "loss": 0.804646110534668,
+      "step": 490,
+      "token_acc": 0.8057767421588184
+    },
+    {
+      "epoch": 0.11872254541137361,
+      "grad_norm": 2.21875,
+      "learning_rate": 1.9591376662327734e-05,
+      "loss": 0.8249678611755371,
+      "step": 500,
+      "token_acc": 0.8001123161700187
+    },
+    {
+      "epoch": 0.1210969963196011,
+      "grad_norm": 1.8671875,
+      "learning_rate": 1.9569333590466613e-05,
+      "loss": 0.8495967864990235,
+      "step": 510,
+      "token_acc": 0.7987011056949234
+    },
+    {
+      "epoch": 0.12347144722782856,
+      "grad_norm": 1.9453125,
+      "learning_rate": 1.9546724546303772e-05,
+      "loss": 0.8431295394897461,
+      "step": 520,
+      "token_acc": 0.7987501511913881
+    },
+    {
+      "epoch": 0.12584589813605604,
+      "grad_norm": 2.1875,
+      "learning_rate": 1.9523550867037118e-05,
+      "loss": 0.7887910366058349,
+      "step": 530,
+      "token_acc": 0.8088436975062688
+    },
+    {
+      "epoch": 0.1282203490442835,
+      "grad_norm": 2.25,
+      "learning_rate": 1.9499813923259534e-05,
+      "loss": 0.8247316360473633,
+      "step": 540,
+      "token_acc": 0.7975765486656078
+    },
+    {
+      "epoch": 0.13059479995251097,
+      "grad_norm": 2.375,
+      "learning_rate": 1.9475515118877846e-05,
+      "loss": 0.7966033935546875,
+      "step": 550,
+      "token_acc": 0.8059755978842846
+    },
+    {
+      "epoch": 0.13296925086073846,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.9450655891029773e-05,
+      "loss": 0.8014211654663086,
+      "step": 560,
+      "token_acc": 0.805231602114903
+    },
+    {
+      "epoch": 0.13534370176896593,
+      "grad_norm": 2.5,
+      "learning_rate": 1.942523770999893e-05,
+      "loss": 0.8178875923156739,
+      "step": 570,
+      "token_acc": 0.8025316896503691
+    },
+    {
+      "epoch": 0.1377181526771934,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.9399262079127875e-05,
+      "loss": 0.8942203521728516,
+      "step": 580,
+      "token_acc": 0.782212068817137
+    },
+    {
+      "epoch": 0.14009260358542086,
+      "grad_norm": 4.96875,
+      "learning_rate": 1.9372730534729186e-05,
+      "loss": 0.8536576271057129,
+      "step": 590,
+      "token_acc": 0.7989557850493375
+    },
+    {
+      "epoch": 0.14246705449364835,
+      "grad_norm": 2.25,
+      "learning_rate": 1.934564464599461e-05,
+      "loss": 0.8536449432373047,
+      "step": 600,
+      "token_acc": 0.795050688046872
+    },
+    {
+      "epoch": 0.14484150540187582,
+      "grad_norm": 2.609375,
+      "learning_rate": 1.931800601490224e-05,
+      "loss": 0.8148602485656739,
+      "step": 610,
+      "token_acc": 0.8029704613342183
+    },
+    {
+      "epoch": 0.14721595631010329,
+      "grad_norm": 2.34375,
+      "learning_rate": 1.9289816276121773e-05,
+      "loss": 0.8230174064636231,
+      "step": 620,
+      "token_acc": 0.8014702368626452
+    },
+    {
+      "epoch": 0.14959040721833075,
+      "grad_norm": 2.09375,
+      "learning_rate": 1.9261077096917836e-05,
+      "loss": 0.7954535484313965,
+      "step": 630,
+      "token_acc": 0.809043463738466
+    },
+    {
+      "epoch": 0.15196485812655824,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.9231790177051354e-05,
+      "loss": 0.8178482055664062,
+      "step": 640,
+      "token_acc": 0.8051890837286508
+    },
+    {
+      "epoch": 0.1543393090347857,
+      "grad_norm": 2.34375,
+      "learning_rate": 1.920195724867905e-05,
+      "loss": 0.8123950958251953,
+      "step": 650,
+      "token_acc": 0.8020062685949756
+    },
+    {
+      "epoch": 0.15671375994301318,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.917158007625097e-05,
+      "loss": 0.7799261093139649,
+      "step": 660,
+      "token_acc": 0.8093542932500546
+    },
+    {
+      "epoch": 0.15908821085124064,
+      "grad_norm": 2.03125,
+      "learning_rate": 1.9140660456406145e-05,
+      "loss": 0.7878777980804443,
+      "step": 670,
+      "token_acc": 0.805255187656039
+    },
+    {
+      "epoch": 0.16146266175946813,
+      "grad_norm": 2.09375,
+      "learning_rate": 1.910920021786632e-05,
+      "loss": 0.8661971092224121,
+      "step": 680,
+      "token_acc": 0.7945319895317111
+    },
+    {
+      "epoch": 0.1638371126676956,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.9077201221327795e-05,
+      "loss": 0.8040632247924805,
+      "step": 690,
+      "token_acc": 0.804484348935335
+    },
+    {
+      "epoch": 0.16621156357592307,
+      "grad_norm": 2.046875,
+      "learning_rate": 1.9044665359351375e-05,
+      "loss": 0.8295541763305664,
+      "step": 700,
+      "token_acc": 0.7984636031283262
+    },
+    {
+      "epoch": 0.16858601448415053,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.901159455625045e-05,
+      "loss": 0.8069572448730469,
+      "step": 710,
+      "token_acc": 0.8055688074089219
+    },
+    {
+      "epoch": 0.17096046539237802,
+      "grad_norm": 2.359375,
+      "learning_rate": 1.8977990767977156e-05,
+      "loss": 0.8333905220031739,
+      "step": 720,
+      "token_acc": 0.7994880473513679
+    },
+    {
+      "epoch": 0.1733349163006055,
+      "grad_norm": 2.453125,
+      "learning_rate": 1.8943855982006723e-05,
+      "loss": 0.8103729248046875,
+      "step": 730,
+      "token_acc": 0.8037981914211142
+    },
+    {
+      "epoch": 0.17570936720883296,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.8909192217219895e-05,
+      "loss": 0.8239153861999512,
+      "step": 740,
+      "token_acc": 0.8019522727099948
+    },
+    {
+      "epoch": 0.17808381811706042,
+      "grad_norm": 2.34375,
+      "learning_rate": 1.8874001523783547e-05,
+      "loss": 0.8832343101501465,
+      "step": 750,
+      "token_acc": 0.7915780485911666
+    },
+    {
+      "epoch": 0.18045826902528792,
+      "grad_norm": 2.109375,
+      "learning_rate": 1.8838285983029428e-05,
+      "loss": 0.861814308166504,
+      "step": 760,
+      "token_acc": 0.7936564279741903
+    },
+    {
+      "epoch": 0.18283271993351538,
+      "grad_norm": 2.328125,
+      "learning_rate": 1.8802047707331047e-05,
+      "loss": 0.8146088600158692,
+      "step": 770,
+      "token_acc": 0.8014844309681289
+    },
+    {
+      "epoch": 0.18520717084174285,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.8765288839978747e-05,
+      "loss": 0.8515184402465821,
+      "step": 780,
+      "token_acc": 0.7958683689872635
+    },
+    {
+      "epoch": 0.1875816217499703,
+      "grad_norm": 2.4375,
+      "learning_rate": 1.872801155505295e-05,
+      "loss": 0.810789966583252,
+      "step": 790,
+      "token_acc": 0.8031616443033751
+    },
+    {
+      "epoch": 0.1899560726581978,
+      "grad_norm": 2.421875,
+      "learning_rate": 1.8690218057295557e-05,
+      "loss": 0.7724108695983887,
+      "step": 800,
+      "token_acc": 0.8105164098341372
+    },
+    {
+      "epoch": 0.19233052356642527,
+      "grad_norm": 2.375,
+      "learning_rate": 1.8651910581979557e-05,
+      "loss": 0.8402803421020508,
+      "step": 810,
+      "token_acc": 0.7986368566756787
+    },
+    {
+      "epoch": 0.19470497447465274,
+      "grad_norm": 2.34375,
+      "learning_rate": 1.861309139477683e-05,
+      "loss": 0.8044181823730469,
+      "step": 820,
+      "token_acc": 0.8063233092088844
+    },
+    {
+      "epoch": 0.1970794253828802,
+      "grad_norm": 2.21875,
+      "learning_rate": 1.8573762791624132e-05,
+      "loss": 0.8206415176391602,
+      "step": 830,
+      "token_acc": 0.8014003819223424
+    },
+    {
+      "epoch": 0.19945387629110767,
+      "grad_norm": 2.09375,
+      "learning_rate": 1.853392709858731e-05,
+      "loss": 0.8116635322570801,
+      "step": 840,
+      "token_acc": 0.8050399145939228
+    },
+    {
+      "epoch": 0.20182832719933516,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.849358667172372e-05,
+      "loss": 0.7985735416412354,
+      "step": 850,
+      "token_acc": 0.8062234558405511
+    },
+    {
+      "epoch": 0.20420277810756263,
+      "grad_norm": 2.34375,
+      "learning_rate": 1.845274389694291e-05,
+      "loss": 0.7977107524871826,
+      "step": 860,
+      "token_acc": 0.8060721363456202
+    },
+    {
+      "epoch": 0.2065772290157901,
+      "grad_norm": 2.0625,
+      "learning_rate": 1.8411401189865465e-05,
+      "loss": 0.8075976371765137,
+      "step": 870,
+      "token_acc": 0.805371624135109
+    },
+    {
+      "epoch": 0.20895167992401756,
+      "grad_norm": 2.0,
+      "learning_rate": 1.836956099568015e-05,
+      "loss": 0.8140823364257812,
+      "step": 880,
+      "token_acc": 0.8000443606521016
+    },
+    {
+      "epoch": 0.21132613083224505,
+      "grad_norm": 2.109375,
+      "learning_rate": 1.832722578899932e-05,
+      "loss": 0.804258918762207,
+      "step": 890,
+      "token_acc": 0.8035196031871669
+    },
+    {
+      "epoch": 0.21370058174047252,
+      "grad_norm": 2.109375,
+      "learning_rate": 1.8284398073712522e-05,
+      "loss": 0.8273359298706054,
+      "step": 900,
+      "token_acc": 0.7991114761442545
+    },
+    {
+      "epoch": 0.21607503264869998,
+      "grad_norm": 2.453125,
+      "learning_rate": 1.8241080382838413e-05,
+      "loss": 0.8466628074645997,
+      "step": 910,
+      "token_acc": 0.7975126607428199
+    },
+    {
+      "epoch": 0.21844948355692745,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.819727527837497e-05,
+      "loss": 0.8572871208190918,
+      "step": 920,
+      "token_acc": 0.7982073512495271
+    },
+    {
+      "epoch": 0.22082393446515494,
+      "grad_norm": 2.328125,
+      "learning_rate": 1.815298535114793e-05,
+      "loss": 0.8341035842895508,
+      "step": 930,
+      "token_acc": 0.8009801568560656
+    },
+    {
+      "epoch": 0.2231983853733824,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.8108213220657578e-05,
+      "loss": 0.8007295608520508,
+      "step": 940,
+      "token_acc": 0.8061111645350515
+    },
+    {
+      "epoch": 0.22557283628160987,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.806296153492381e-05,
+      "loss": 0.7931745052337646,
+      "step": 950,
+      "token_acc": 0.8095192045644357
+    },
+    {
+      "epoch": 0.22794728718983734,
+      "grad_norm": 2.09375,
+      "learning_rate": 1.801723297032952e-05,
+      "loss": 0.801950740814209,
+      "step": 960,
+      "token_acc": 0.8049479719685708
+    },
+    {
+      "epoch": 0.23032173809806483,
+      "grad_norm": 2.78125,
+      "learning_rate": 1.7971030231462314e-05,
+      "loss": 0.7776188850402832,
+      "step": 970,
+      "token_acc": 0.8142859443961212
+    },
+    {
+      "epoch": 0.2326961890062923,
+      "grad_norm": 1.859375,
+      "learning_rate": 1.7924356050954518e-05,
+      "loss": 0.8285764694213867,
+      "step": 980,
+      "token_acc": 0.8021880261445598
+    },
+    {
+      "epoch": 0.23507063991451976,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.7877213189321612e-05,
+      "loss": 0.8198677062988281,
+      "step": 990,
+      "token_acc": 0.8001236729252662
+    },
+    {
+      "epoch": 0.23744509082274723,
+      "grad_norm": 2.203125,
+      "learning_rate": 1.7829604434798912e-05,
+      "loss": 0.7920489311218262,
+      "step": 1000,
+      "token_acc": 0.8073018352206169
+    },
+    {
+      "epoch": 0.23981954173097472,
+      "grad_norm": 2.3125,
+      "learning_rate": 1.7781532603176685e-05,
+      "loss": 0.8316179275512695,
+      "step": 1010,
+      "token_acc": 0.8009123998237224
+    },
+    {
+      "epoch": 0.2421939926392022,
+      "grad_norm": 2.265625,
+      "learning_rate": 1.7733000537633605e-05,
+      "loss": 0.8350128173828125,
+      "step": 1020,
+      "token_acc": 0.801742308346082
+    },
+    {
+      "epoch": 0.24456844354742965,
+      "grad_norm": 2.3125,
+      "learning_rate": 1.7684011108568593e-05,
+      "loss": 0.7910916805267334,
+      "step": 1030,
+      "token_acc": 0.8070748948149743
+    },
+    {
+      "epoch": 0.24694289445565712,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.7634567213431054e-05,
+      "loss": 0.8148965835571289,
+      "step": 1040,
+      "token_acc": 0.8031462065482154
+    },
+    {
+      "epoch": 0.2493173453638846,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.7584671776549518e-05,
+      "loss": 0.8274960517883301,
+      "step": 1050,
+      "token_acc": 0.7996730180935606
+    },
+    {
+      "epoch": 0.2516917962721121,
+      "grad_norm": 1.984375,
+      "learning_rate": 1.7534327748958654e-05,
+      "loss": 0.8012187004089355,
+      "step": 1060,
+      "token_acc": 0.8048769339762781
+    },
+    {
+      "epoch": 0.25406624718033954,
+      "grad_norm": 2.0625,
+      "learning_rate": 1.7483538108224756e-05,
+      "loss": 0.7923438549041748,
+      "step": 1070,
+      "token_acc": 0.8064775160599572
+    },
+    {
+      "epoch": 0.256440698088567,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.7432305858269635e-05,
+      "loss": 0.7856131553649902,
+      "step": 1080,
+      "token_acc": 0.8090334396311439
+    },
+    {
+      "epoch": 0.2588151489967945,
+      "grad_norm": 2.3125,
+      "learning_rate": 1.7380634029192954e-05,
+      "loss": 0.7943347930908203,
+      "step": 1090,
+      "token_acc": 0.807943280210817
+    },
+    {
+      "epoch": 0.26118959990502194,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.7328525677092995e-05,
+      "loss": 0.7980467319488526,
+      "step": 1100,
+      "token_acc": 0.8047425745882195
+    },
+    {
+      "epoch": 0.26356405081324946,
+      "grad_norm": 2.1875,
+      "learning_rate": 1.727598388388594e-05,
+      "loss": 0.8072722434997559,
+      "step": 1110,
+      "token_acc": 0.804585309759954
+    },
+    {
+      "epoch": 0.2659385017214769,
+      "grad_norm": 2.375,
+      "learning_rate": 1.7223011757123562e-05,
+      "loss": 0.7909623622894287,
+      "step": 1120,
+      "token_acc": 0.8086852874567921
+    },
+    {
+      "epoch": 0.2683129526297044,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.7169612429809463e-05,
+      "loss": 0.7782119274139404,
+      "step": 1130,
+      "token_acc": 0.8113280193988646
+    },
+    {
+      "epoch": 0.27068740353793186,
+      "grad_norm": 2.109375,
+      "learning_rate": 1.711578906021374e-05,
+      "loss": 0.8028268814086914,
+      "step": 1140,
+      "token_acc": 0.8041270673050203
+    },
+    {
+      "epoch": 0.2730618544461593,
+      "grad_norm": 2.40625,
+      "learning_rate": 1.7061544831686214e-05,
+      "loss": 0.8485577583312989,
+      "step": 1150,
+      "token_acc": 0.7969658046022025
+    },
+    {
+      "epoch": 0.2754363053543868,
+      "grad_norm": 2.03125,
+      "learning_rate": 1.7006882952468148e-05,
+      "loss": 0.834971809387207,
+      "step": 1160,
+      "token_acc": 0.7993655354043215
+    },
+    {
+      "epoch": 0.27781075626261426,
+      "grad_norm": 2.25,
+      "learning_rate": 1.69518066555025e-05,
+      "loss": 0.8089464187622071,
+      "step": 1170,
+      "token_acc": 0.8011860614310974
+    },
+    {
+      "epoch": 0.2801852071708417,
+      "grad_norm": 2.03125,
+      "learning_rate": 1.6896319198242696e-05,
+      "loss": 0.8391291618347168,
+      "step": 1180,
+      "token_acc": 0.7987688958390826
+    },
+    {
+      "epoch": 0.28255965807906924,
+      "grad_norm": 2.109375,
+      "learning_rate": 1.684042386245999e-05,
+      "loss": 0.7904163837432862,
+      "step": 1190,
+      "token_acc": 0.8074876664814534
+    },
+    {
+      "epoch": 0.2849341089872967,
+      "grad_norm": 1.96875,
+      "learning_rate": 1.678412395404937e-05,
+      "loss": 0.7824483871459961,
+      "step": 1200,
+      "token_acc": 0.8093992893181143
+    },
+    {
+      "epoch": 0.2873085598955242,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.6727422802834e-05,
+      "loss": 0.8354628562927247,
+      "step": 1210,
+      "token_acc": 0.8000748287124005
+    },
+    {
+      "epoch": 0.28968301080375164,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.6670323762368316e-05,
+      "loss": 0.7695659160614013,
+      "step": 1220,
+      "token_acc": 0.8124942094047689
+    },
+    {
+      "epoch": 0.2920574617119791,
+      "grad_norm": 2.03125,
+      "learning_rate": 1.661283020973966e-05,
+      "loss": 0.7790391921997071,
+      "step": 1230,
+      "token_acc": 0.8085999484096896
+    },
+    {
+      "epoch": 0.29443191262020657,
+      "grad_norm": 2.265625,
+      "learning_rate": 1.655494554536855e-05,
+      "loss": 0.8052985191345214,
+      "step": 1240,
+      "token_acc": 0.8059546745278274
+    },
+    {
+      "epoch": 0.29680636352843404,
+      "grad_norm": 2.671875,
+      "learning_rate": 1.6496673192807572e-05,
+      "loss": 0.8351935386657715,
+      "step": 1250,
+      "token_acc": 0.7982909279975848
+    },
+    {
+      "epoch": 0.2991808144366615,
+      "grad_norm": 2.34375,
+      "learning_rate": 1.6438016598538883e-05,
+      "loss": 0.8197705268859863,
+      "step": 1260,
+      "token_acc": 0.8024019921489961
+    },
+    {
+      "epoch": 0.30155526534488897,
+      "grad_norm": 1.96875,
+      "learning_rate": 1.637897923177039e-05,
+      "loss": 0.7983882427215576,
+      "step": 1270,
+      "token_acc": 0.8050125468570898
+    },
+    {
+      "epoch": 0.3039297162531165,
+      "grad_norm": 2.125,
+      "learning_rate": 1.6319564584230533e-05,
+      "loss": 0.7988826274871826,
+      "step": 1280,
+      "token_acc": 0.8073074256177509
+    },
+    {
+      "epoch": 0.30630416716134395,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.6259776169961807e-05,
+      "loss": 0.7942438125610352,
+      "step": 1290,
+      "token_acc": 0.8063025679493113
+    },
+    {
+      "epoch": 0.3086786180695714,
+      "grad_norm": 2.28125,
+      "learning_rate": 1.61996175251129e-05,
+      "loss": 0.8152091979980469,
+      "step": 1300,
+      "token_acc": 0.8010248393260379
+    },
+    {
+      "epoch": 0.3110530689777989,
+      "grad_norm": 2.25,
+      "learning_rate": 1.6139092207729563e-05,
+      "loss": 0.8136390686035156,
+      "step": 1310,
+      "token_acc": 0.8025988655107213
+    },
+    {
+      "epoch": 0.31342751988602635,
+      "grad_norm": 2.25,
+      "learning_rate": 1.607820379754416e-05,
+      "loss": 0.760651683807373,
+      "step": 1320,
+      "token_acc": 0.8136050427107323
+    },
+    {
+      "epoch": 0.3158019707942538,
+      "grad_norm": 2.03125,
+      "learning_rate": 1.601695589576396e-05,
+      "loss": 0.8023890495300293,
+      "step": 1330,
+      "token_acc": 0.8075189789190333
+    },
+    {
+      "epoch": 0.3181764217024813,
+      "grad_norm": 2.40625,
+      "learning_rate": 1.595535212485813e-05,
+      "loss": 0.7719482421875,
+      "step": 1340,
+      "token_acc": 0.8137801459943943
+    },
+    {
+      "epoch": 0.32055087261070875,
+      "grad_norm": 2.53125,
+      "learning_rate": 1.589339612834351e-05,
+      "loss": 0.8081175804138183,
+      "step": 1350,
+      "token_acc": 0.8057676062947783
+    },
+    {
+      "epoch": 0.32292532351893627,
+      "grad_norm": 2.28125,
+      "learning_rate": 1.5831091570569098e-05,
+      "loss": 0.8052699089050293,
+      "step": 1360,
+      "token_acc": 0.8064986866426695
+    },
+    {
+      "epoch": 0.32529977442716373,
+      "grad_norm": 2.09375,
+      "learning_rate": 1.576844213649933e-05,
+      "loss": 0.808250617980957,
+      "step": 1370,
+      "token_acc": 0.8039683907338568
+    },
+    {
+      "epoch": 0.3276742253353912,
+      "grad_norm": 2.265625,
+      "learning_rate": 1.5705451531496135e-05,
+      "loss": 0.789212989807129,
+      "step": 1380,
+      "token_acc": 0.8090594203927695
+    },
+    {
+      "epoch": 0.33004867624361867,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.5642123481099793e-05,
+      "loss": 0.8304048538208008,
+      "step": 1390,
+      "token_acc": 0.8008559043159094
+    },
+    {
+      "epoch": 0.33242312715184613,
+      "grad_norm": 2.203125,
+      "learning_rate": 1.5578461730808575e-05,
+      "loss": 0.8099192619323731,
+      "step": 1400,
+      "token_acc": 0.8035508807697951
+    },
+    {
+      "epoch": 0.3347975780600736,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.551447004585723e-05,
+      "loss": 0.8090396881103515,
+      "step": 1410,
+      "token_acc": 0.8046525395692222
+    },
+    {
+      "epoch": 0.33717202896830106,
+      "grad_norm": 2.25,
+      "learning_rate": 1.5450152210994286e-05,
+      "loss": 0.7888967990875244,
+      "step": 1420,
+      "token_acc": 0.8090682610861983
+    },
+    {
+      "epoch": 0.33954647987652853,
+      "grad_norm": 2.109375,
+      "learning_rate": 1.5385512030258202e-05,
+      "loss": 0.7790110111236572,
+      "step": 1430,
+      "token_acc": 0.8098405954510606
+    },
+    {
+      "epoch": 0.34192093078475605,
+      "grad_norm": 2.21875,
+      "learning_rate": 1.532055332675239e-05,
+      "loss": 0.7999553680419922,
+      "step": 1440,
+      "token_acc": 0.8052976884965928
+    },
+    {
+      "epoch": 0.3442953816929835,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.5255279942419093e-05,
+      "loss": 0.8073784828186035,
+      "step": 1450,
+      "token_acc": 0.8043324793172102
+    },
+    {
+      "epoch": 0.346669832601211,
+      "grad_norm": 2.671875,
+      "learning_rate": 1.5189695737812153e-05,
+      "loss": 0.7951379299163819,
+      "step": 1460,
+      "token_acc": 0.8099502131165156
+    },
+    {
+      "epoch": 0.34904428350943845,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.5123804591868685e-05,
+      "loss": 0.8613236427307129,
+      "step": 1470,
+      "token_acc": 0.7953647972994491
+    },
+    {
+      "epoch": 0.3514187344176659,
+      "grad_norm": 1.96875,
+      "learning_rate": 1.5057610401679661e-05,
+      "loss": 0.7715866565704346,
+      "step": 1480,
+      "token_acc": 0.8117064235460638
+    },
+    {
+      "epoch": 0.3537931853258934,
+      "grad_norm": 2.40625,
+      "learning_rate": 1.4991117082259414e-05,
+      "loss": 0.822512149810791,
+      "step": 1490,
+      "token_acc": 0.8022701513434229
+    },
+    {
+      "epoch": 0.35616763623412084,
+      "grad_norm": 2.203125,
+      "learning_rate": 1.4924328566314094e-05,
+      "loss": 0.7812392234802246,
+      "step": 1500,
+      "token_acc": 0.8102505324717519
+    },
+    {
+      "epoch": 0.3585420871423483,
+      "grad_norm": 2.03125,
+      "learning_rate": 1.4857248804009063e-05,
+      "loss": 0.810976219177246,
+      "step": 1510,
+      "token_acc": 0.8046864276082586
+    },
+    {
+      "epoch": 0.36091653805057583,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.4789881762735268e-05,
+      "loss": 0.8119425773620605,
+      "step": 1520,
+      "token_acc": 0.8023746575778812
+    },
+    {
+      "epoch": 0.3632909889588033,
+      "grad_norm": 1.890625,
+      "learning_rate": 1.4722231426874598e-05,
+      "loss": 0.8455527305603028,
+      "step": 1530,
+      "token_acc": 0.7973304825427813
+    },
+    {
+      "epoch": 0.36566543986703076,
+      "grad_norm": 2.265625,
+      "learning_rate": 1.4654301797564218e-05,
+      "loss": 0.829925537109375,
+      "step": 1540,
+      "token_acc": 0.8005760067880686
+    },
+    {
+      "epoch": 0.3680398907752582,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.4586096892459934e-05,
+      "loss": 0.7579694271087647,
+      "step": 1550,
+      "token_acc": 0.815877350394008
+    },
+    {
+      "epoch": 0.3704143416834857,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.4517620745498567e-05,
+      "loss": 0.8355647087097168,
+      "step": 1560,
+      "token_acc": 0.7986828197414424
+    },
+    {
+      "epoch": 0.37278879259171316,
+      "grad_norm": 2.203125,
+      "learning_rate": 1.4448877406659371e-05,
+      "loss": 0.7629793643951416,
+      "step": 1570,
+      "token_acc": 0.8142376359889035
+    },
+    {
+      "epoch": 0.3751632434999406,
+      "grad_norm": 2.265625,
+      "learning_rate": 1.4379870941724496e-05,
+      "loss": 0.7802756309509278,
+      "step": 1580,
+      "token_acc": 0.8120543578558845
+    },
+    {
+      "epoch": 0.3775376944081681,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.4310605432038527e-05,
+      "loss": 0.7767571449279785,
+      "step": 1590,
+      "token_acc": 0.8114644692567756
+    },
+    {
+      "epoch": 0.3799121453163956,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.4241084974267079e-05,
+      "loss": 0.8257313728332519,
+      "step": 1600,
+      "token_acc": 0.79900317747086
+    },
+    {
+      "epoch": 0.3822865962246231,
+      "grad_norm": 2.203125,
+      "learning_rate": 1.4171313680154527e-05,
+      "loss": 0.7681684970855713,
+      "step": 1610,
+      "token_acc": 0.8119423286180631
+    },
+    {
+      "epoch": 0.38466104713285054,
+      "grad_norm": 1.9609375,
+      "learning_rate": 1.4101295676280796e-05,
+      "loss": 0.8211341857910156,
+      "step": 1620,
+      "token_acc": 0.801680723488225
+    },
+    {
+      "epoch": 0.387035498041078,
+      "grad_norm": 1.921875,
+      "learning_rate": 1.4031035103817314e-05,
+      "loss": 0.7744133949279786,
+      "step": 1630,
+      "token_acc": 0.8111924686192469
+    },
+    {
+      "epoch": 0.3894099489493055,
+      "grad_norm": 2.203125,
+      "learning_rate": 1.396053611828207e-05,
+      "loss": 0.8150890350341797,
+      "step": 1640,
+      "token_acc": 0.8025460080254601
+    },
+    {
+      "epoch": 0.39178439985753294,
+      "grad_norm": 1.9921875,
+      "learning_rate": 1.3889802889293851e-05,
+      "loss": 0.7851222038269043,
+      "step": 1650,
+      "token_acc": 0.8113327026367188
+    },
+    {
+      "epoch": 0.3941588507657604,
+      "grad_norm": 2.65625,
+      "learning_rate": 1.3818839600325628e-05,
+      "loss": 0.8212462425231933,
+      "step": 1660,
+      "token_acc": 0.8027813848280484
+    },
+    {
+      "epoch": 0.39653330167398787,
+      "grad_norm": 2.296875,
+      "learning_rate": 1.3747650448457125e-05,
+      "loss": 0.8306264877319336,
+      "step": 1670,
+      "token_acc": 0.8011434924164218
+    },
+    {
+      "epoch": 0.39890775258221534,
+      "grad_norm": 2.28125,
+      "learning_rate": 1.367623964412659e-05,
+      "loss": 0.7631672382354736,
+      "step": 1680,
+      "token_acc": 0.8128248304737633
+    },
+    {
+      "epoch": 0.40128220349044286,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.3604611410881765e-05,
+      "loss": 0.8237265586853028,
+      "step": 1690,
+      "token_acc": 0.8013616562056133
+    },
+    {
+      "epoch": 0.4036566543986703,
+      "grad_norm": 2.046875,
+      "learning_rate": 1.3532769985130096e-05,
+      "loss": 0.8060081481933594,
+      "step": 1700,
+      "token_acc": 0.8063578930425335
+    },
+    {
+      "epoch": 0.4060311053068978,
+      "grad_norm": 2.28125,
+      "learning_rate": 1.3460719615888161e-05,
+      "loss": 0.834253978729248,
+      "step": 1710,
+      "token_acc": 0.7984603735487128
+    },
+    {
+      "epoch": 0.40840555621512525,
+      "grad_norm": 1.9609375,
+      "learning_rate": 1.3388464564530368e-05,
+      "loss": 0.7958603382110596,
+      "step": 1720,
+      "token_acc": 0.8083800459733033
+    },
+    {
+      "epoch": 0.4107800071233527,
+      "grad_norm": 1.9453125,
+      "learning_rate": 1.3316009104536936e-05,
+      "loss": 0.7930515289306641,
+      "step": 1730,
+      "token_acc": 0.8063091253092709
+    },
+    {
+      "epoch": 0.4131544580315802,
+      "grad_norm": 2.03125,
+      "learning_rate": 1.3243357521241116e-05,
+      "loss": 0.7910933017730712,
+      "step": 1740,
+      "token_acc": 0.8070608037029244
+    },
+    {
+      "epoch": 0.41552890893980765,
+      "grad_norm": 2.53125,
+      "learning_rate": 1.3170514111575756e-05,
+      "loss": 0.8198260307312012,
+      "step": 1750,
+      "token_acc": 0.8018118722680103
+    },
+    {
+      "epoch": 0.4179033598480351,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.3097483183819153e-05,
+      "loss": 0.8183247566223144,
+      "step": 1760,
+      "token_acc": 0.8039341492930064
+    },
+    {
+      "epoch": 0.42027781075626264,
+      "grad_norm": 2.296875,
+      "learning_rate": 1.3024269057340245e-05,
+      "loss": 0.8238384246826171,
+      "step": 1770,
+      "token_acc": 0.8012736039487817
+    },
+    {
+      "epoch": 0.4226522616644901,
+      "grad_norm": 2.546875,
+      "learning_rate": 1.2950876062343147e-05,
+      "loss": 0.7632989883422852,
+      "step": 1780,
+      "token_acc": 0.8137166717554732
+    },
+    {
+      "epoch": 0.42502671257271757,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.287730853961104e-05,
+      "loss": 0.8405383110046387,
+      "step": 1790,
+      "token_acc": 0.7962870597209794
+    },
+    {
+      "epoch": 0.42740116348094503,
+      "grad_norm": 2.59375,
+      "learning_rate": 1.2803570840249443e-05,
+      "loss": 0.7838696479797364,
+      "step": 1800,
+      "token_acc": 0.8097151443878218
+    },
+    {
+      "epoch": 0.4297756143891725,
+      "grad_norm": 2.328125,
+      "learning_rate": 1.2729667325428863e-05,
+      "loss": 0.7929567337036133,
+      "step": 1810,
+      "token_acc": 0.8071844660194175
+    },
+    {
+      "epoch": 0.43215006529739997,
+      "grad_norm": 1.9921875,
+      "learning_rate": 1.2655602366126856e-05,
+      "loss": 0.8084596633911133,
+      "step": 1820,
+      "token_acc": 0.8045593426478004
+    },
+    {
+      "epoch": 0.43452451620562743,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.2581380342869518e-05,
+      "loss": 0.7986243724822998,
+      "step": 1830,
+      "token_acc": 0.8068372657588001
+    },
+    {
+      "epoch": 0.4368989671138549,
+      "grad_norm": 2.203125,
+      "learning_rate": 1.2507005645472392e-05,
+      "loss": 0.7939201831817627,
+      "step": 1840,
+      "token_acc": 0.8094208521137198
+    },
+    {
+      "epoch": 0.4392734180220824,
+      "grad_norm": 1.9765625,
+      "learning_rate": 1.243248267278084e-05,
+      "loss": 0.8393895149230957,
+      "step": 1850,
+      "token_acc": 0.7988877383288558
+    },
+    {
+      "epoch": 0.4416478689303099,
+      "grad_norm": 2.03125,
+      "learning_rate": 1.2357815832409879e-05,
+      "loss": 0.789722204208374,
+      "step": 1860,
+      "token_acc": 0.808149762386116
+    },
+    {
+      "epoch": 0.44402231983853735,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.2283009540483479e-05,
+      "loss": 0.8282936096191407,
+      "step": 1870,
+      "token_acc": 0.8030894677945676
+    },
+    {
+      "epoch": 0.4463967707467648,
+      "grad_norm": 2.25,
+      "learning_rate": 1.2208068221373392e-05,
+      "loss": 0.7985751628875732,
+      "step": 1880,
+      "token_acc": 0.8051834882989714
+    },
+    {
+      "epoch": 0.4487712216549923,
+      "grad_norm": 1.9765625,
+      "learning_rate": 1.213299630743747e-05,
+      "loss": 0.8041236877441407,
+      "step": 1890,
+      "token_acc": 0.8058205481248361
+    },
+    {
+      "epoch": 0.45114567256321975,
+      "grad_norm": 2.578125,
+      "learning_rate": 1.2057798238757515e-05,
+      "loss": 0.8212913513183594,
+      "step": 1900,
+      "token_acc": 0.8039757896110676
+    },
+    {
+      "epoch": 0.4535201234714472,
+      "grad_norm": 2.0625,
+      "learning_rate": 1.1982478462876674e-05,
+      "loss": 0.7796264171600342,
+      "step": 1910,
+      "token_acc": 0.8096826259531927
+    },
+    {
+      "epoch": 0.4558945743796747,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.1907041434536388e-05,
+      "loss": 0.7350980281829834,
+      "step": 1920,
+      "token_acc": 0.8176107626081524
+    },
+    {
+      "epoch": 0.4582690252879022,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.183149161541292e-05,
+      "loss": 0.7631175518035889,
+      "step": 1930,
+      "token_acc": 0.8158383171787997
+    },
+    {
+      "epoch": 0.46064347619612966,
+      "grad_norm": 2.03125,
+      "learning_rate": 1.1755833473853479e-05,
+      "loss": 0.8004861831665039,
+      "step": 1940,
+      "token_acc": 0.8053857373386296
+    },
+    {
+      "epoch": 0.46301792710435713,
+      "grad_norm": 1.9765625,
+      "learning_rate": 1.168007148461194e-05,
+      "loss": 0.77386474609375,
+      "step": 1950,
+      "token_acc": 0.8103342108098954
+    },
+    {
+      "epoch": 0.4653923780125846,
+      "grad_norm": 1.96875,
+      "learning_rate": 1.1604210128584178e-05,
+      "loss": 0.7687146663665771,
+      "step": 1960,
+      "token_acc": 0.812569867097255
+    },
+    {
+      "epoch": 0.46776682892081206,
+      "grad_norm": 2.28125,
+      "learning_rate": 1.1528253892543053e-05,
+      "loss": 0.7979438781738282,
+      "step": 1970,
+      "token_acc": 0.8045926859617654
+    },
+    {
+      "epoch": 0.4701412798290395,
+      "grad_norm": 2.359375,
+      "learning_rate": 1.145220726887305e-05,
+      "loss": 0.7988301277160644,
+      "step": 1980,
+      "token_acc": 0.8058375531370142
+    },
+    {
+      "epoch": 0.472515730737267,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.1376074755304576e-05,
+      "loss": 0.7771591186523438,
+      "step": 1990,
+      "token_acc": 0.809505599562961
+    },
+    {
+      "epoch": 0.47489018164549446,
+      "grad_norm": 2.28125,
+      "learning_rate": 1.129986085464794e-05,
+      "loss": 0.7955132484436035,
+      "step": 2000,
+      "token_acc": 0.8072251119997594
+    },
+    {
+      "epoch": 0.477264632553722,
+      "grad_norm": 2.109375,
+      "learning_rate": 1.1223570074527034e-05,
+      "loss": 0.8154989242553711,
+      "step": 2010,
+      "token_acc": 0.805622868591042
+    },
+    {
+      "epoch": 0.47963908346194944,
+      "grad_norm": 2.3125,
+      "learning_rate": 1.1147206927112757e-05,
+      "loss": 0.7790208339691163,
+      "step": 2020,
+      "token_acc": 0.8109172101768611
+    },
+    {
+      "epoch": 0.4820135343701769,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.1070775928856097e-05,
+      "loss": 0.8297024726867676,
+      "step": 2030,
+      "token_acc": 0.7964178592852857
+    },
+    {
+      "epoch": 0.4843879852784044,
+      "grad_norm": 2.453125,
+      "learning_rate": 1.0994281600221064e-05,
+      "loss": 0.8055323600769043,
+      "step": 2040,
+      "token_acc": 0.8031506735844937
+    },
+    {
+      "epoch": 0.48676243618663184,
+      "grad_norm": 2.0625,
+      "learning_rate": 1.0917728465417292e-05,
+      "loss": 0.8090975761413575,
+      "step": 2050,
+      "token_acc": 0.8059850532172038
+    },
+    {
+      "epoch": 0.4891368870948593,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.0841121052132475e-05,
+      "loss": 0.8202219009399414,
+      "step": 2060,
+      "token_acc": 0.8019910327532487
+    },
+    {
+      "epoch": 0.4915113380030868,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.076446389126457e-05,
+      "loss": 0.8387545585632324,
+      "step": 2070,
+      "token_acc": 0.797718417474917
+    },
+    {
+      "epoch": 0.49388578891131424,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.0687761516653826e-05,
+      "loss": 0.8033242225646973,
+      "step": 2080,
+      "token_acc": 0.8046132526465114
+    },
+    {
+      "epoch": 0.4962602398195417,
+      "grad_norm": 2.109375,
+      "learning_rate": 1.0611018464814622e-05,
+      "loss": 0.8189582824707031,
+      "step": 2090,
+      "token_acc": 0.8013748762180858
+    },
+    {
+      "epoch": 0.4986346907277692,
+      "grad_norm": 2.59375,
+      "learning_rate": 1.0534239274667182e-05,
+      "loss": 0.8109881401062011,
+      "step": 2100,
+      "token_acc": 0.8055181919327787
+    },
+    {
+      "epoch": 0.5010091416359966,
+      "grad_norm": 2.3125,
+      "learning_rate": 1.0457428487269099e-05,
+      "loss": 0.8230169296264649,
+      "step": 2110,
+      "token_acc": 0.8018922875899309
+    },
+    {
+      "epoch": 0.5033835925442242,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.0380590645546768e-05,
+      "loss": 0.7921685695648193,
+      "step": 2120,
+      "token_acc": 0.8078819708936374
+    },
+    {
+      "epoch": 0.5057580434524516,
+      "grad_norm": 2.265625,
+      "learning_rate": 1.0303730294026693e-05,
+      "loss": 0.7888332843780518,
+      "step": 2130,
+      "token_acc": 0.8091902773387459
+    },
+    {
+      "epoch": 0.5081324943606791,
+      "grad_norm": 2.046875,
+      "learning_rate": 1.0226851978566706e-05,
+      "loss": 0.7957067966461182,
+      "step": 2140,
+      "token_acc": 0.8063656156044965
+    },
+    {
+      "epoch": 0.5105069452689066,
+      "grad_norm": 1.859375,
+      "learning_rate": 1.0149960246087103e-05,
+      "loss": 0.7647091388702393,
+      "step": 2150,
+      "token_acc": 0.8139527092110663
+    },
+    {
+      "epoch": 0.512881396177134,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.007305964430173e-05,
+      "loss": 0.8165877342224122,
+      "step": 2160,
+      "token_acc": 0.8007157938872999
+    },
+    {
+      "epoch": 0.5152558470853615,
+      "grad_norm": 2.28125,
+      "learning_rate": 9.996154721448998e-06,
+      "loss": 0.8375003814697266,
+      "step": 2170,
+      "token_acc": 0.7968574089823027
+    },
+    {
+      "epoch": 0.517630297993589,
+      "grad_norm": 2.34375,
+      "learning_rate": 9.919250026022882e-06,
+      "loss": 0.8013765335083007,
+      "step": 2180,
+      "token_acc": 0.8054499263742132
+    },
+    {
+      "epoch": 0.5200047489018165,
+      "grad_norm": 2.03125,
+      "learning_rate": 9.842350106503906e-06,
+      "loss": 0.7803740501403809,
+      "step": 2190,
+      "token_acc": 0.8098679687637717
+    },
+    {
+      "epoch": 0.5223791998100439,
+      "grad_norm": 2.078125,
+      "learning_rate": 9.765459511090129e-06,
+      "loss": 0.8000008583068847,
+      "step": 2200,
+      "token_acc": 0.8057989742761378
+    },
+    {
+      "epoch": 0.5247536507182714,
+      "grad_norm": 2.140625,
+      "learning_rate": 9.68858278742814e-06,
+      "loss": 0.7937568664550781,
+      "step": 2210,
+      "token_acc": 0.8075620860246865
+    },
+    {
+      "epoch": 0.5271281016264989,
+      "grad_norm": 2.15625,
+      "learning_rate": 9.611724482344096e-06,
+      "loss": 0.7942758083343506,
+      "step": 2220,
+      "token_acc": 0.8073675233683929
+    },
+    {
+      "epoch": 0.5295025525347263,
+      "grad_norm": 2.375,
+      "learning_rate": 9.534889141574792e-06,
+      "loss": 0.787619161605835,
+      "step": 2230,
+      "token_acc": 0.8090135403568239
+    },
+    {
+      "epoch": 0.5318770034429539,
+      "grad_norm": 2.296875,
+      "learning_rate": 9.458081309498815e-06,
+      "loss": 0.7825831890106201,
+      "step": 2240,
+      "token_acc": 0.8100791747429285
+    },
+    {
+      "epoch": 0.5342514543511813,
+      "grad_norm": 2.28125,
+      "learning_rate": 9.381305528867768e-06,
+      "loss": 0.7672498226165771,
+      "step": 2250,
+      "token_acc": 0.8137144945957846
+    },
+    {
+      "epoch": 0.5366259052594088,
+      "grad_norm": 3.203125,
+      "learning_rate": 9.304566340537595e-06,
+      "loss": 0.8283431053161621,
+      "step": 2260,
+      "token_acc": 0.7992323291971314
+    },
+    {
+      "epoch": 0.5390003561676362,
+      "grad_norm": 2.15625,
+      "learning_rate": 9.22786828320001e-06,
+      "loss": 0.7935975074768067,
+      "step": 2270,
+      "token_acc": 0.8065179588125734
+    },
+    {
+      "epoch": 0.5413748070758637,
+      "grad_norm": 2.0,
+      "learning_rate": 9.151215893114048e-06,
+      "loss": 0.7871967792510987,
+      "step": 2280,
+      "token_acc": 0.8085648862399922
+    },
+    {
+      "epoch": 0.5437492579840911,
+      "grad_norm": 2.046875,
+      "learning_rate": 9.074613703837797e-06,
+      "loss": 0.7708443641662598,
+      "step": 2290,
+      "token_acc": 0.8130785186642726
+    },
+    {
+      "epoch": 0.5461237088923186,
+      "grad_norm": 2.125,
+      "learning_rate": 8.998066245960245e-06,
+      "loss": 0.7797691822052002,
+      "step": 2300,
+      "token_acc": 0.8125213195224427
+    },
+    {
+      "epoch": 0.5484981598005462,
+      "grad_norm": 2.203125,
+      "learning_rate": 8.921578046833322e-06,
+      "loss": 0.8027665138244628,
+      "step": 2310,
+      "token_acc": 0.8040712468193384
+    },
+    {
+      "epoch": 0.5508726107087736,
+      "grad_norm": 2.265625,
+      "learning_rate": 8.84515363030414e-06,
+      "loss": 0.8279308319091797,
+      "step": 2320,
+      "token_acc": 0.7988508193232603
+    },
+    {
+      "epoch": 0.5532470616170011,
+      "grad_norm": 2.015625,
+      "learning_rate": 8.76879751644743e-06,
+      "loss": 0.7780996799468994,
+      "step": 2330,
+      "token_acc": 0.8096343537414966
+    },
+    {
+      "epoch": 0.5556215125252285,
+      "grad_norm": 2.5,
+      "learning_rate": 8.692514221298198e-06,
+      "loss": 0.7974294185638428,
+      "step": 2340,
+      "token_acc": 0.8056538523403571
+    },
+    {
+      "epoch": 0.557995963433456,
+      "grad_norm": 2.3125,
+      "learning_rate": 8.616308256584636e-06,
+      "loss": 0.8067178726196289,
+      "step": 2350,
+      "token_acc": 0.8050933017685559
+    },
+    {
+      "epoch": 0.5603704143416834,
+      "grad_norm": 2.046875,
+      "learning_rate": 8.540184129461278e-06,
+      "loss": 0.7718284130096436,
+      "step": 2360,
+      "token_acc": 0.8139766414563951
+    },
+    {
+      "epoch": 0.562744865249911,
+      "grad_norm": 2.046875,
+      "learning_rate": 8.46414634224242e-06,
+      "loss": 0.7615556240081787,
+      "step": 2370,
+      "token_acc": 0.8156892359526319
+    },
+    {
+      "epoch": 0.5651193161581385,
+      "grad_norm": 2.0625,
+      "learning_rate": 8.388199392135844e-06,
+      "loss": 0.7809924602508544,
+      "step": 2380,
+      "token_acc": 0.808885738282183
+    },
+    {
+      "epoch": 0.5674937670663659,
+      "grad_norm": 2.234375,
+      "learning_rate": 8.312347770976814e-06,
+      "loss": 0.7736612796783447,
+      "step": 2390,
+      "token_acc": 0.8119160216654939
+    },
+    {
+      "epoch": 0.5698682179745934,
+      "grad_norm": 2.140625,
+      "learning_rate": 8.236595964962435e-06,
+      "loss": 0.7930184364318847,
+      "step": 2400,
+      "token_acc": 0.8080044848514393
+    },
+    {
+      "epoch": 0.5722426688828208,
+      "grad_norm": 2.453125,
+      "learning_rate": 8.160948454386306e-06,
+      "loss": 0.7554466247558593,
+      "step": 2410,
+      "token_acc": 0.8143985058154343
+    },
+    {
+      "epoch": 0.5746171197910483,
+      "grad_norm": 2.390625,
+      "learning_rate": 8.085409713373534e-06,
+      "loss": 0.7986351013183594,
+      "step": 2420,
+      "token_acc": 0.807332629025316
+    },
+    {
+      "epoch": 0.5769915706992758,
+      "grad_norm": 2.296875,
+      "learning_rate": 8.009984209616117e-06,
+      "loss": 0.7782696723937989,
+      "step": 2430,
+      "token_acc": 0.8116431629807126
+    },
+    {
+      "epoch": 0.5793660216075033,
+      "grad_norm": 1.984375,
+      "learning_rate": 7.934676404108707e-06,
+      "loss": 0.8118492126464844,
+      "step": 2440,
+      "token_acc": 0.8057577362487706
+    },
+    {
+      "epoch": 0.5817404725157307,
+      "grad_norm": 2.125,
+      "learning_rate": 7.859490750884764e-06,
+      "loss": 0.7904249668121338,
+      "step": 2450,
+      "token_acc": 0.807311019401078
+    },
+    {
+      "epoch": 0.5841149234239582,
+      "grad_norm": 2.09375,
+      "learning_rate": 7.784431696753134e-06,
+      "loss": 0.7790792465209961,
+      "step": 2460,
+      "token_acc": 0.808160842205248
+    },
+    {
+      "epoch": 0.5864893743321857,
+      "grad_norm": 2.203125,
+      "learning_rate": 7.709503681035028e-06,
+      "loss": 0.7816617488861084,
+      "step": 2470,
+      "token_acc": 0.8110952253082612
+    },
+    {
+      "epoch": 0.5888638252404131,
+      "grad_norm": 2.21875,
+      "learning_rate": 7.634711135301485e-06,
+      "loss": 0.783286714553833,
+      "step": 2480,
+      "token_acc": 0.8100758021250373
+    },
+    {
+      "epoch": 0.5912382761486407,
+      "grad_norm": 2.109375,
+      "learning_rate": 7.5600584831112376e-06,
+      "loss": 0.8032812118530274,
+      "step": 2490,
+      "token_acc": 0.8055206730920165
+    },
+    {
+      "epoch": 0.5936127270568681,
+      "grad_norm": 2.125,
+      "learning_rate": 7.485550139749111e-06,
+      "loss": 0.8193548202514649,
+      "step": 2500,
+      "token_acc": 0.800822783219244
+    },
+    {
+      "epoch": 0.5959871779650956,
+      "grad_norm": 2.21875,
+      "learning_rate": 7.411190511964878e-06,
+      "loss": 0.7791476726531983,
+      "step": 2510,
+      "token_acc": 0.8121993914209993
+    },
+    {
+      "epoch": 0.598361628873323,
+      "grad_norm": 2.3125,
+      "learning_rate": 7.336983997712617e-06,
+      "loss": 0.8094292640686035,
+      "step": 2520,
+      "token_acc": 0.8035865736531247
+    },
+    {
+      "epoch": 0.6007360797815505,
+      "grad_norm": 2.421875,
+      "learning_rate": 7.2629349858906e-06,
+      "loss": 0.8156153678894043,
+      "step": 2530,
+      "token_acc": 0.8034735688390524
+    },
+    {
+      "epoch": 0.6031105306897779,
+      "grad_norm": 2.09375,
+      "learning_rate": 7.189047856081719e-06,
+      "loss": 0.7990139484405517,
+      "step": 2540,
+      "token_acc": 0.8063241755985381
+    },
+    {
+      "epoch": 0.6054849815980055,
+      "grad_norm": 2.234375,
+      "learning_rate": 7.11532697829445e-06,
+      "loss": 0.8318069458007813,
+      "step": 2550,
+      "token_acc": 0.7985742196961134
+    },
+    {
+      "epoch": 0.607859432506233,
+      "grad_norm": 2.046875,
+      "learning_rate": 7.041776712704399e-06,
+      "loss": 0.8214270591735839,
+      "step": 2560,
+      "token_acc": 0.799731214119804
+    },
+    {
+      "epoch": 0.6102338834144604,
+      "grad_norm": 2.046875,
+      "learning_rate": 6.968401409396421e-06,
+      "loss": 0.80093994140625,
+      "step": 2570,
+      "token_acc": 0.8062186172267285
+    },
+    {
+      "epoch": 0.6126083343226879,
+      "grad_norm": 2.203125,
+      "learning_rate": 6.895205408107336e-06,
+      "loss": 0.7696633815765381,
+      "step": 2580,
+      "token_acc": 0.8130642765016082
+    },
+    {
+      "epoch": 0.6149827852309153,
+      "grad_norm": 2.265625,
+      "learning_rate": 6.822193037969251e-06,
+      "loss": 0.778245210647583,
+      "step": 2590,
+      "token_acc": 0.811267918481322
+    },
+    {
+      "epoch": 0.6173572361391428,
+      "grad_norm": 2.34375,
+      "learning_rate": 6.749368617253528e-06,
+      "loss": 0.7724869251251221,
+      "step": 2600,
+      "token_acc": 0.8125995780045644
+    },
+    {
+      "epoch": 0.6197316870473702,
+      "grad_norm": 1.9453125,
+      "learning_rate": 6.676736453115371e-06,
+      "loss": 0.8026960372924805,
+      "step": 2610,
+      "token_acc": 0.8053778634125455
+    },
+    {
+      "epoch": 0.6221061379555978,
+      "grad_norm": 2.0,
+      "learning_rate": 6.604300841339097e-06,
+      "loss": 0.7808023452758789,
+      "step": 2620,
+      "token_acc": 0.8105542292224127
+    },
+    {
+      "epoch": 0.6244805888638253,
+      "grad_norm": 2.1875,
+      "learning_rate": 6.532066066084049e-06,
+      "loss": 0.7848945617675781,
+      "step": 2630,
+      "token_acc": 0.810422657026455
+    },
+    {
+      "epoch": 0.6268550397720527,
+      "grad_norm": 2.15625,
+      "learning_rate": 6.460036399631222e-06,
+      "loss": 0.8058065414428711,
+      "step": 2640,
+      "token_acc": 0.8050107662290213
+    },
+    {
+      "epoch": 0.6292294906802802,
+      "grad_norm": 1.9453125,
+      "learning_rate": 6.388216102130572e-06,
+      "loss": 0.8067639350891114,
+      "step": 2650,
+      "token_acc": 0.8063436896042097
+    },
+    {
+      "epoch": 0.6316039415885076,
+      "grad_norm": 2.28125,
+      "learning_rate": 6.316609421349065e-06,
+      "loss": 0.8017507553100586,
+      "step": 2660,
+      "token_acc": 0.8056435527396457
+    },
+    {
+      "epoch": 0.6339783924967352,
+      "grad_norm": 2.125,
+      "learning_rate": 6.245220592419436e-06,
+      "loss": 0.7701010704040527,
+      "step": 2670,
+      "token_acc": 0.8119448485036056
+    },
+    {
+      "epoch": 0.6363528434049626,
+      "grad_norm": 2.40625,
+      "learning_rate": 6.174053837589709e-06,
+      "loss": 0.831716251373291,
+      "step": 2680,
+      "token_acc": 0.7989679015826031
+    },
+    {
+      "epoch": 0.6387272943131901,
+      "grad_norm": 2.171875,
+      "learning_rate": 6.103113365973466e-06,
+      "loss": 0.7996344566345215,
+      "step": 2690,
+      "token_acc": 0.8051077923459078
+    },
+    {
+      "epoch": 0.6411017452214175,
+      "grad_norm": 2.53125,
+      "learning_rate": 6.032403373300922e-06,
+      "loss": 0.7853100299835205,
+      "step": 2700,
+      "token_acc": 0.8093361781203758
+    },
+    {
+      "epoch": 0.643476196129645,
+      "grad_norm": 2.453125,
+      "learning_rate": 5.96192804167074e-06,
+      "loss": 0.7865749359130859,
+      "step": 2710,
+      "token_acc": 0.8098021922640797
+    },
+    {
+      "epoch": 0.6458506470378725,
+      "grad_norm": 2.234375,
+      "learning_rate": 5.891691539302717e-06,
+      "loss": 0.7574759960174561,
+      "step": 2720,
+      "token_acc": 0.8139308087736987
+    },
+    {
+      "epoch": 0.6482250979461,
+      "grad_norm": 2.125,
+      "learning_rate": 5.821698020291234e-06,
+      "loss": 0.8135777473449707,
+      "step": 2730,
+      "token_acc": 0.8045498042627229
+    },
+    {
+      "epoch": 0.6505995488543275,
+      "grad_norm": 2.28125,
+      "learning_rate": 5.751951624359572e-06,
+      "loss": 0.7769800186157226,
+      "step": 2740,
+      "token_acc": 0.8130432683021055
+    },
+    {
+      "epoch": 0.6529739997625549,
+      "grad_norm": 2.40625,
+      "learning_rate": 5.6824564766150724e-06,
+      "loss": 0.7626114845275879,
+      "step": 2750,
+      "token_acc": 0.8144680214685698
+    },
+    {
+      "epoch": 0.6553484506707824,
+      "grad_norm": 2.34375,
+      "learning_rate": 5.613216687305159e-06,
+      "loss": 0.7772722244262695,
+      "step": 2760,
+      "token_acc": 0.8119055860488882
+    },
+    {
+      "epoch": 0.6577229015790098,
+      "grad_norm": 2.0625,
+      "learning_rate": 5.544236351574237e-06,
+      "loss": 0.7662568092346191,
+      "step": 2770,
+      "token_acc": 0.8119369544584649
+    },
+    {
+      "epoch": 0.6600973524872373,
+      "grad_norm": 2.171875,
+      "learning_rate": 5.475519549221493e-06,
+      "loss": 0.7982632637023925,
+      "step": 2780,
+      "token_acc": 0.8086979278186187
+    },
+    {
+      "epoch": 0.6624718033954649,
+      "grad_norm": 2.234375,
+      "learning_rate": 5.407070344459593e-06,
+      "loss": 0.8140118598937989,
+      "step": 2790,
+      "token_acc": 0.8034894169329073
+    },
+    {
+      "epoch": 0.6648462543036923,
+      "grad_norm": 2.1875,
+      "learning_rate": 5.33889278567431e-06,
+      "loss": 0.7451247692108154,
+      "step": 2800,
+      "token_acc": 0.819039335766227
+    },
+    {
+      "epoch": 0.6672207052119198,
+      "grad_norm": 2.15625,
+      "learning_rate": 5.2709909051850915e-06,
+      "loss": 0.8014011383056641,
+      "step": 2810,
+      "token_acc": 0.8035785127312473
+    },
+    {
+      "epoch": 0.6695951561201472,
+      "grad_norm": 2.3125,
+      "learning_rate": 5.203368719006545e-06,
+      "loss": 0.794771957397461,
+      "step": 2820,
+      "token_acc": 0.8053081878256237
+    },
+    {
+      "epoch": 0.6719696070283747,
+      "grad_norm": 2.265625,
+      "learning_rate": 5.136030226610953e-06,
+      "loss": 0.8222715377807617,
+      "step": 2830,
+      "token_acc": 0.8009289064655569
+    },
+    {
+      "epoch": 0.6743440579366021,
+      "grad_norm": 2.4375,
+      "learning_rate": 5.068979410691687e-06,
+      "loss": 0.7945393562316895,
+      "step": 2840,
+      "token_acc": 0.8073549776888752
+    },
+    {
+      "epoch": 0.6767185088448296,
+      "grad_norm": 2.28125,
+      "learning_rate": 5.002220236927681e-06,
+      "loss": 0.7931512355804443,
+      "step": 2850,
+      "token_acc": 0.8088140469222757
+    },
+    {
+      "epoch": 0.6790929597530571,
+      "grad_norm": 2.359375,
+      "learning_rate": 4.935756653748874e-06,
+      "loss": 0.8048454284667969,
+      "step": 2860,
+      "token_acc": 0.8060467721841791
+    },
+    {
+      "epoch": 0.6814674106612846,
+      "grad_norm": 2.34375,
+      "learning_rate": 4.869592592102684e-06,
+      "loss": 0.801272201538086,
+      "step": 2870,
+      "token_acc": 0.8062680948401746
+    },
+    {
+      "epoch": 0.6838418615695121,
+      "grad_norm": 1.9375,
+      "learning_rate": 4.803731965221511e-06,
+      "loss": 0.7873353958129883,
+      "step": 2880,
+      "token_acc": 0.8068211444417233
+    },
+    {
+      "epoch": 0.6862163124777395,
+      "grad_norm": 2.09375,
+      "learning_rate": 4.7381786683912936e-06,
+      "loss": 0.7926517486572265,
+      "step": 2890,
+      "token_acc": 0.8089569016171474
+    },
+    {
+      "epoch": 0.688590763385967,
+      "grad_norm": 2.171875,
+      "learning_rate": 4.672936578721127e-06,
+      "loss": 0.8082955360412598,
+      "step": 2900,
+      "token_acc": 0.8034763020912605
+    },
+    {
+      "epoch": 0.6909652142941944,
+      "grad_norm": 2.34375,
+      "learning_rate": 4.608009554913955e-06,
+      "loss": 0.7745423316955566,
+      "step": 2910,
+      "token_acc": 0.8131339444164049
+    },
+    {
+      "epoch": 0.693339665202422,
+      "grad_norm": 2.609375,
+      "learning_rate": 4.543401437038335e-06,
+      "loss": 0.7683454513549804,
+      "step": 2920,
+      "token_acc": 0.8135946658722718
+    },
+    {
+      "epoch": 0.6957141161106494,
+      "grad_norm": 2.28125,
+      "learning_rate": 4.479116046301346e-06,
+      "loss": 0.8194952964782715,
+      "step": 2930,
+      "token_acc": 0.8029072288754712
+    },
+    {
+      "epoch": 0.6980885670188769,
+      "grad_norm": 2.234375,
+      "learning_rate": 4.415157184822555e-06,
+      "loss": 0.801603126525879,
+      "step": 2940,
+      "token_acc": 0.8056845805751898
+    },
+    {
+      "epoch": 0.7004630179271043,
+      "grad_norm": 2.0625,
+      "learning_rate": 4.3515286354091715e-06,
+      "loss": 0.786121416091919,
+      "step": 2950,
+      "token_acc": 0.8103164800931425
+    },
+    {
+      "epoch": 0.7028374688353318,
+      "grad_norm": 2.28125,
+      "learning_rate": 4.288234161332294e-06,
+      "loss": 0.7889712810516357,
+      "step": 2960,
+      "token_acc": 0.8065297731222008
+    },
+    {
+      "epoch": 0.7052119197435593,
+      "grad_norm": 1.984375,
+      "learning_rate": 4.22527750610435e-06,
+      "loss": 0.8059993743896484,
+      "step": 2970,
+      "token_acc": 0.8038128707072207
+    },
+    {
+      "epoch": 0.7075863706517868,
+      "grad_norm": 2.359375,
+      "learning_rate": 4.162662393257674e-06,
+      "loss": 0.799560546875,
+      "step": 2980,
+      "token_acc": 0.8066369272521918
+    },
+    {
+      "epoch": 0.7099608215600143,
+      "grad_norm": 2.015625,
+      "learning_rate": 4.10039252612429e-06,
+      "loss": 0.7870376586914063,
+      "step": 2990,
+      "token_acc": 0.8095627460942462
+    },
+    {
+      "epoch": 0.7123352724682417,
+      "grad_norm": 2.1875,
+      "learning_rate": 4.038471587616881e-06,
+      "loss": 0.7926857471466064,
+      "step": 3000,
+      "token_acc": 0.8098951775371444
+    },
+    {
+      "epoch": 0.7147097233764692,
+      "grad_norm": 2.46875,
+      "learning_rate": 3.9769032400109605e-06,
+      "loss": 0.8438973426818848,
+      "step": 3010,
+      "token_acc": 0.7926301731161751
+    },
+    {
+      "epoch": 0.7170841742846966,
+      "grad_norm": 2.203125,
+      "learning_rate": 3.915691124728275e-06,
+      "loss": 0.817176342010498,
+      "step": 3020,
+      "token_acc": 0.8012067501081748
+    },
+    {
+      "epoch": 0.7194586251929241,
+      "grad_norm": 2.234375,
+      "learning_rate": 3.854838862121421e-06,
+      "loss": 0.8113916397094727,
+      "step": 3030,
+      "token_acc": 0.80450205227911
+    },
+    {
+      "epoch": 0.7218330761011517,
+      "grad_norm": 2.09375,
+      "learning_rate": 3.7943500512597454e-06,
+      "loss": 0.761334753036499,
+      "step": 3040,
+      "token_acc": 0.8140671339983823
+    },
+    {
+      "epoch": 0.7242075270093791,
+      "grad_norm": 1.984375,
+      "learning_rate": 3.7342282697164545e-06,
+      "loss": 0.7908918380737304,
+      "step": 3050,
+      "token_acc": 0.8073891827613072
+    },
+    {
+      "epoch": 0.7265819779176066,
+      "grad_norm": 2.140625,
+      "learning_rate": 3.674477073357041e-06,
+      "loss": 0.806179141998291,
+      "step": 3060,
+      "token_acc": 0.8040053229065699
+    },
+    {
+      "epoch": 0.728956428825834,
+      "grad_norm": 2.015625,
+      "learning_rate": 3.6150999961289666e-06,
+      "loss": 0.7566771030426025,
+      "step": 3070,
+      "token_acc": 0.8142128142128142
+    },
+    {
+      "epoch": 0.7313308797340615,
+      "grad_norm": 2.296875,
+      "learning_rate": 3.5561005498526457e-06,
+      "loss": 0.8143975257873535,
+      "step": 3080,
+      "token_acc": 0.8036463548864963
+    },
+    {
+      "epoch": 0.7337053306422889,
+      "grad_norm": 2.109375,
+      "learning_rate": 3.497482224013745e-06,
+      "loss": 0.8138324737548828,
+      "step": 3090,
+      "token_acc": 0.8023479205108929
+    },
+    {
+      "epoch": 0.7360797815505165,
+      "grad_norm": 2.171875,
+      "learning_rate": 3.439248485556801e-06,
+      "loss": 0.7907161712646484,
+      "step": 3100,
+      "token_acc": 0.8091813863113042
+    },
+    {
+      "epoch": 0.7384542324587439,
+      "grad_norm": 2.109375,
+      "learning_rate": 3.3814027786801675e-06,
+      "loss": 0.8030494689941406,
+      "step": 3110,
+      "token_acc": 0.8059615596198278
+    },
+    {
+      "epoch": 0.7408286833669714,
+      "grad_norm": 1.9921875,
+      "learning_rate": 3.3239485246323113e-06,
+      "loss": 0.8017807006835938,
+      "step": 3120,
+      "token_acc": 0.8057972166998012
+    },
+    {
+      "epoch": 0.7432031342751989,
+      "grad_norm": 2.21875,
+      "learning_rate": 3.266889121509458e-06,
+      "loss": 0.764946699142456,
+      "step": 3130,
+      "token_acc": 0.8120940227019224
+    },
+    {
+      "epoch": 0.7455775851834263,
+      "grad_norm": 2.1875,
+      "learning_rate": 3.2102279440546323e-06,
+      "loss": 0.8018248558044434,
+      "step": 3140,
+      "token_acc": 0.8088556926505179
+    },
+    {
+      "epoch": 0.7479520360916538,
+      "grad_norm": 1.8515625,
+      "learning_rate": 3.153968343458035e-06,
+      "loss": 0.7989253520965576,
+      "step": 3150,
+      "token_acc": 0.8043635486981678
+    },
+    {
+      "epoch": 0.7503264869998812,
+      "grad_norm": 2.28125,
+      "learning_rate": 3.0981136471588624e-06,
+      "loss": 0.7952893257141114,
+      "step": 3160,
+      "token_acc": 0.8086431305827307
+    },
+    {
+      "epoch": 0.7527009379081088,
+      "grad_norm": 2.3125,
+      "learning_rate": 3.0426671586484945e-06,
+      "loss": 0.7887931346893311,
+      "step": 3170,
+      "token_acc": 0.8084571976237885
+    },
+    {
+      "epoch": 0.7550753888163362,
+      "grad_norm": 2.171875,
+      "learning_rate": 2.9876321572751143e-06,
+      "loss": 0.8174541473388672,
+      "step": 3180,
+      "token_acc": 0.8040116660711764
+    },
+    {
+      "epoch": 0.7574498397245637,
+      "grad_norm": 1.8203125,
+      "learning_rate": 2.9330118980497546e-06,
+      "loss": 0.7841171741485595,
+      "step": 3190,
+      "token_acc": 0.8102224741165295
+    },
+    {
+      "epoch": 0.7598242906327912,
+      "grad_norm": 2.203125,
+      "learning_rate": 2.878809611453779e-06,
+      "loss": 0.8041182518005371,
+      "step": 3200,
+      "token_acc": 0.8085118255227224
+    },
+    {
+      "epoch": 0.7621987415410186,
+      "grad_norm": 2.3125,
+      "learning_rate": 2.8250285032478188e-06,
+      "loss": 0.832216453552246,
+      "step": 3210,
+      "token_acc": 0.7993245599857086
+    },
+    {
+      "epoch": 0.7645731924492462,
+      "grad_norm": 2.140625,
+      "learning_rate": 2.7716717542821737e-06,
+      "loss": 0.7945997238159179,
+      "step": 3220,
+      "token_acc": 0.8095869771707445
+    },
+    {
+      "epoch": 0.7669476433574736,
+      "grad_norm": 2.09375,
+      "learning_rate": 2.718742520308679e-06,
+      "loss": 0.8016815185546875,
+      "step": 3230,
+      "token_acc": 0.806536672209066
+    },
+    {
+      "epoch": 0.7693220942657011,
+      "grad_norm": 2.140625,
+      "learning_rate": 2.6662439317940538e-06,
+      "loss": 0.7698002338409424,
+      "step": 3240,
+      "token_acc": 0.8114276209090531
+    },
+    {
+      "epoch": 0.7716965451739285,
+      "grad_norm": 2.3125,
+      "learning_rate": 2.6141790937347686e-06,
+      "loss": 0.7474682807922364,
+      "step": 3250,
+      "token_acc": 0.8172773903714631
+    },
+    {
+      "epoch": 0.774070996082156,
+      "grad_norm": 2.328125,
+      "learning_rate": 2.5625510854733836e-06,
+      "loss": 0.8135623931884766,
+      "step": 3260,
+      "token_acc": 0.8064711074356508
+    },
+    {
+      "epoch": 0.7764454469903834,
+      "grad_norm": 2.5,
+      "learning_rate": 2.511362960516437e-06,
+      "loss": 0.824278450012207,
+      "step": 3270,
+      "token_acc": 0.8002157765882402
+    },
+    {
+      "epoch": 0.778819897898611,
+      "grad_norm": 2.203125,
+      "learning_rate": 2.4606177463538404e-06,
+      "loss": 0.788828992843628,
+      "step": 3280,
+      "token_acc": 0.8074596774193549
+    },
+    {
+      "epoch": 0.7811943488068385,
+      "grad_norm": 2.609375,
+      "learning_rate": 2.41031844427982e-06,
+      "loss": 0.8141343116760253,
+      "step": 3290,
+      "token_acc": 0.8028626098977519
+    },
+    {
+      "epoch": 0.7835687997150659,
+      "grad_norm": 2.40625,
+      "learning_rate": 2.360468029215409e-06,
+      "loss": 0.8015660285949707,
+      "step": 3300,
+      "token_acc": 0.803906339234723
+    },
+    {
+      "epoch": 0.7859432506232934,
+      "grad_norm": 2.03125,
+      "learning_rate": 2.311069449532498e-06,
+      "loss": 0.8212325096130371,
+      "step": 3310,
+      "token_acc": 0.8036264341304444
+    },
+    {
+      "epoch": 0.7883177015315208,
+      "grad_norm": 2.421875,
+      "learning_rate": 2.2621256268794513e-06,
+      "loss": 0.8013825416564941,
+      "step": 3320,
+      "token_acc": 0.8058569855892992
+    },
+    {
+      "epoch": 0.7906921524397483,
+      "grad_norm": 2.25,
+      "learning_rate": 2.213639456008314e-06,
+      "loss": 0.7905919075012207,
+      "step": 3330,
+      "token_acc": 0.8071378216425802
+    },
+    {
+      "epoch": 0.7930666033479757,
+      "grad_norm": 2.484375,
+      "learning_rate": 2.1656138046035933e-06,
+      "loss": 0.7892219066619873,
+      "step": 3340,
+      "token_acc": 0.8098729269139272
+    },
+    {
+      "epoch": 0.7954410542562033,
+      "grad_norm": 2.375,
+      "learning_rate": 2.118051513112668e-06,
+      "loss": 0.8001435279846192,
+      "step": 3350,
+      "token_acc": 0.805480238873544
+    },
+    {
+      "epoch": 0.7978155051644307,
+      "grad_norm": 2.078125,
+      "learning_rate": 2.070955394577776e-06,
+      "loss": 0.764467191696167,
+      "step": 3360,
+      "token_acc": 0.8152371121121121
+    },
+    {
+      "epoch": 0.8001899560726582,
+      "grad_norm": 2.34375,
+      "learning_rate": 2.0243282344696437e-06,
+      "loss": 0.7969135761260986,
+      "step": 3370,
+      "token_acc": 0.8051913209698515
+    },
+    {
+      "epoch": 0.8025644069808857,
+      "grad_norm": 2.53125,
+      "learning_rate": 1.9781727905227453e-06,
+      "loss": 0.752310037612915,
+      "step": 3380,
+      "token_acc": 0.8162480625465631
+    },
+    {
+      "epoch": 0.8049388578891131,
+      "grad_norm": 2.203125,
+      "learning_rate": 1.9324917925721966e-06,
+      "loss": 0.8153909683227539,
+      "step": 3390,
+      "token_acc": 0.8055264322825656
+    },
+    {
+      "epoch": 0.8073133087973406,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.887287942392293e-06,
+      "loss": 0.8089158058166503,
+      "step": 3400,
+      "token_acc": 0.8052709140135028
+    },
+    {
+      "epoch": 0.809687759705568,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.842563913536728e-06,
+      "loss": 0.7809914112091064,
+      "step": 3410,
+      "token_acc": 0.8090842749340412
+    },
+    {
+      "epoch": 0.8120622106137956,
+      "grad_norm": 2.640625,
+      "learning_rate": 1.798322351180456e-06,
+      "loss": 0.7919546604156494,
+      "step": 3420,
+      "token_acc": 0.8062131789117299
+    },
+    {
+      "epoch": 0.814436661522023,
+      "grad_norm": 2.046875,
+      "learning_rate": 1.7545658719632498e-06,
+      "loss": 0.8009761810302735,
+      "step": 3430,
+      "token_acc": 0.8060019137492989
+    },
+    {
+      "epoch": 0.8168111124302505,
+      "grad_norm": 2.59375,
+      "learning_rate": 1.7112970638349435e-06,
+      "loss": 0.8127578735351563,
+      "step": 3440,
+      "token_acc": 0.8041406450509025
+    },
+    {
+      "epoch": 0.819185563338478,
+      "grad_norm": 2.3125,
+      "learning_rate": 1.6685184859023618e-06,
+      "loss": 0.8158830642700196,
+      "step": 3450,
+      "token_acc": 0.802465601304808
+    },
+    {
+      "epoch": 0.8215600142467054,
+      "grad_norm": 2.671875,
+      "learning_rate": 1.6262326682779716e-06,
+      "loss": 0.7651051998138427,
+      "step": 3460,
+      "token_acc": 0.8130639926837756
+    },
+    {
+      "epoch": 0.823934465154933,
+      "grad_norm": 2.1875,
+      "learning_rate": 1.584442111930241e-06,
+      "loss": 0.7686656475067138,
+      "step": 3470,
+      "token_acc": 0.8124111014744145
+    },
+    {
+      "epoch": 0.8263089160631604,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.543149288535708e-06,
+      "loss": 0.794713020324707,
+      "step": 3480,
+      "token_acc": 0.8102111141917852
+    },
+    {
+      "epoch": 0.8286833669713879,
+      "grad_norm": 2.328125,
+      "learning_rate": 1.5023566403328105e-06,
+      "loss": 0.8148051261901855,
+      "step": 3490,
+      "token_acc": 0.8029095728734399
+    },
+    {
+      "epoch": 0.8310578178796153,
+      "grad_norm": 2.6875,
+      "learning_rate": 1.4620665799774326e-06,
+      "loss": 0.8246870040893555,
+      "step": 3500,
+      "token_acc": 0.8012668975161483
+    },
+    {
+      "epoch": 0.8334322687878428,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.4222814904002124e-06,
+      "loss": 0.7918514728546142,
+      "step": 3510,
+      "token_acc": 0.8071518823547321
+    },
+    {
+      "epoch": 0.8358067196960702,
+      "grad_norm": 2.34375,
+      "learning_rate": 1.3830037246656047e-06,
+      "loss": 0.8109148025512696,
+      "step": 3520,
+      "token_acc": 0.8034993270524899
+    },
+    {
+      "epoch": 0.8381811706042978,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.3442356058327088e-06,
+      "loss": 0.7884156703948975,
+      "step": 3530,
+      "token_acc": 0.8079911209766926
+    },
+    {
+      "epoch": 0.8405556215125253,
+      "grad_norm": 2.1875,
+      "learning_rate": 1.305979426817876e-06,
+      "loss": 0.8220973968505859,
+      "step": 3540,
+      "token_acc": 0.8011534975694158
+    },
+    {
+      "epoch": 0.8429300724207527,
+      "grad_norm": 2.046875,
+      "learning_rate": 1.268237450259091e-06,
+      "loss": 0.7788082122802734,
+      "step": 3550,
+      "token_acc": 0.8128685496170754
+    },
+    {
+      "epoch": 0.8453045233289802,
+      "grad_norm": 2.46875,
+      "learning_rate": 1.2310119083821558e-06,
+      "loss": 0.7843327045440673,
+      "step": 3560,
+      "token_acc": 0.8103702681013729
+    },
+    {
+      "epoch": 0.8476789742372076,
+      "grad_norm": 2.1875,
+      "learning_rate": 1.1943050028686665e-06,
+      "loss": 0.7872900485992431,
+      "step": 3570,
+      "token_acc": 0.807969943855798
+    },
+    {
+      "epoch": 0.8500534251454351,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.1581189047257846e-06,
+      "loss": 0.7901401042938232,
+      "step": 3580,
+      "token_acc": 0.809362291252055
+    },
+    {
+      "epoch": 0.8524278760536625,
+      "grad_norm": 2.0,
+      "learning_rate": 1.1224557541578485e-06,
+      "loss": 0.819882583618164,
+      "step": 3590,
+      "token_acc": 0.80215800196182
+    },
+    {
+      "epoch": 0.8548023269618901,
+      "grad_norm": 2.34375,
+      "learning_rate": 1.0873176604397872e-06,
+      "loss": 0.7800666332244873,
+      "step": 3600,
+      "token_acc": 0.8099443463191922
+    },
+    {
+      "epoch": 0.8571767778701176,
+      "grad_norm": 2.359375,
+      "learning_rate": 1.0527067017923654e-06,
+      "loss": 0.7662560939788818,
+      "step": 3610,
+      "token_acc": 0.8134932867516098
+    },
+    {
+      "epoch": 0.859551228778345,
+      "grad_norm": 2.265625,
+      "learning_rate": 1.0186249252592717e-06,
+      "loss": 0.7882128238677979,
+      "step": 3620,
+      "token_acc": 0.8090702687638829
+    },
+    {
+      "epoch": 0.8619256796865725,
+      "grad_norm": 2.15625,
+      "learning_rate": 9.850743465860469e-07,
+      "loss": 0.8053848266601562,
+      "step": 3630,
+      "token_acc": 0.8048451541246556
+    },
+    {
+      "epoch": 0.8643001305947999,
+      "grad_norm": 2.09375,
+      "learning_rate": 9.520569501008658e-07,
+      "loss": 0.8009872436523438,
+      "step": 3640,
+      "token_acc": 0.8047875192707286
+    },
+    {
+      "epoch": 0.8666745815030275,
+      "grad_norm": 2.1875,
+      "learning_rate": 9.195746885971657e-07,
+      "loss": 0.7896884918212891,
+      "step": 3650,
+      "token_acc": 0.8093442295580577
+    },
+    {
+      "epoch": 0.8690490324112549,
+      "grad_norm": 2.421875,
+      "learning_rate": 8.876294832181653e-07,
+      "loss": 0.8029258728027344,
+      "step": 3660,
+      "token_acc": 0.8046547761110354
+    },
+    {
+      "epoch": 0.8714234833194824,
+      "grad_norm": 1.9765625,
+      "learning_rate": 8.562232233432288e-07,
+      "loss": 0.8221800804138184,
+      "step": 3670,
+      "token_acc": 0.8029502033863459
+    },
+    {
+      "epoch": 0.8737979342277098,
+      "grad_norm": 2.296875,
+      "learning_rate": 8.253577664761259e-07,
+      "loss": 0.7743802547454834,
+      "step": 3680,
+      "token_acc": 0.8138093144687379
+    },
+    {
+      "epoch": 0.8761723851359373,
+      "grad_norm": 2.140625,
+      "learning_rate": 7.950349381351597e-07,
+      "loss": 0.7925952911376953,
+      "step": 3690,
+      "token_acc": 0.8071323617012827
+    },
+    {
+      "epoch": 0.8785468360441648,
+      "grad_norm": 2.375,
+      "learning_rate": 7.652565317452132e-07,
+      "loss": 0.8323602676391602,
+      "step": 3700,
+      "token_acc": 0.8002790012382639
+    },
+    {
+      "epoch": 0.8809212869523922,
+      "grad_norm": 2.140625,
+      "learning_rate": 7.360243085316677e-07,
+      "loss": 0.810945987701416,
+      "step": 3710,
+      "token_acc": 0.805199608972496
+    },
+    {
+      "epoch": 0.8832957378606198,
+      "grad_norm": 1.9296875,
+      "learning_rate": 7.073399974162409e-07,
+      "loss": 0.8336031913757325,
+      "step": 3720,
+      "token_acc": 0.8002083500280471
+    },
+    {
+      "epoch": 0.8856701887688472,
+      "grad_norm": 2.109375,
+      "learning_rate": 6.792052949147254e-07,
+      "loss": 0.7754337310791015,
+      "step": 3730,
+      "token_acc": 0.8111203023372747
+    },
+    {
+      "epoch": 0.8880446396770747,
+      "grad_norm": 2.03125,
+      "learning_rate": 6.51621865036659e-07,
+      "loss": 0.7848072528839112,
+      "step": 3740,
+      "token_acc": 0.8085772385587314
+    },
+    {
+      "epoch": 0.8904190905853021,
+      "grad_norm": 2.28125,
+      "learning_rate": 6.245913391868974e-07,
+      "loss": 0.7556170463562012,
+      "step": 3750,
+      "token_acc": 0.8149270549670061
+    },
+    {
+      "epoch": 0.8927935414935296,
+      "grad_norm": 2.078125,
+      "learning_rate": 5.9811531606913e-07,
+      "loss": 0.7851991176605224,
+      "step": 3760,
+      "token_acc": 0.8084357555095388
+    },
+    {
+      "epoch": 0.895167992401757,
+      "grad_norm": 2.0,
+      "learning_rate": 5.72195361591329e-07,
+      "loss": 0.7584320068359375,
+      "step": 3770,
+      "token_acc": 0.8163054217611461
+    },
+    {
+      "epoch": 0.8975424433099846,
+      "grad_norm": 1.9453125,
+      "learning_rate": 5.468330087731289e-07,
+      "loss": 0.8144121170043945,
+      "step": 3780,
+      "token_acc": 0.7994342222185989
+    },
+    {
+      "epoch": 0.8999168942182121,
+      "grad_norm": 2.390625,
+      "learning_rate": 5.220297576551637e-07,
+      "loss": 0.8074371337890625,
+      "step": 3790,
+      "token_acc": 0.8033076359877687
+    },
+    {
+      "epoch": 0.9022913451264395,
+      "grad_norm": 2.125,
+      "learning_rate": 4.977870752103397e-07,
+      "loss": 0.8092558860778809,
+      "step": 3800,
+      "token_acc": 0.8057843504305503
+    },
+    {
+      "epoch": 0.904665796034667,
+      "grad_norm": 2.03125,
+      "learning_rate": 4.741063952570779e-07,
+      "loss": 0.7783875465393066,
+      "step": 3810,
+      "token_acc": 0.8125029569934867
+    },
+    {
+      "epoch": 0.9070402469428944,
+      "grad_norm": 2.25,
+      "learning_rate": 4.5098911837451077e-07,
+      "loss": 0.806202507019043,
+      "step": 3820,
+      "token_acc": 0.8049962217184513
+    },
+    {
+      "epoch": 0.909414697851122,
+      "grad_norm": 2.234375,
+      "learning_rate": 4.2843661181964703e-07,
+      "loss": 0.8107380867004395,
+      "step": 3830,
+      "token_acc": 0.8027301376154053
+    },
+    {
+      "epoch": 0.9117891487593494,
+      "grad_norm": 2.0625,
+      "learning_rate": 4.0645020944650284e-07,
+      "loss": 0.8015091896057129,
+      "step": 3840,
+      "token_acc": 0.8067592396871596
+    },
+    {
+      "epoch": 0.9141635996675769,
+      "grad_norm": 1.9296875,
+      "learning_rate": 3.850312116272148e-07,
+      "loss": 0.7666542053222656,
+      "step": 3850,
+      "token_acc": 0.8130752241949809
+    },
+    {
+      "epoch": 0.9165380505758044,
+      "grad_norm": 2.46875,
+      "learning_rate": 3.6418088517512497e-07,
+      "loss": 0.7932547569274903,
+      "step": 3860,
+      "token_acc": 0.8074404471742874
+    },
+    {
+      "epoch": 0.9189125014840318,
+      "grad_norm": 2.125,
+      "learning_rate": 3.4390046326986506e-07,
+      "loss": 0.7836908340454102,
+      "step": 3870,
+      "token_acc": 0.808913133346514
+    },
+    {
+      "epoch": 0.9212869523922593,
+      "grad_norm": 2.0625,
+      "learning_rate": 3.2419114538441263e-07,
+      "loss": 0.7940024375915528,
+      "step": 3880,
+      "token_acc": 0.8074350001252808
+    },
+    {
+      "epoch": 0.9236614033004867,
+      "grad_norm": 1.8984375,
+      "learning_rate": 3.050540972141547e-07,
+      "loss": 0.8047228813171386,
+      "step": 3890,
+      "token_acc": 0.8029794305675672
+    },
+    {
+      "epoch": 0.9260358542087143,
+      "grad_norm": 2.28125,
+      "learning_rate": 2.864904506079391e-07,
+      "loss": 0.7952239513397217,
+      "step": 3900,
+      "token_acc": 0.8075733433311463
+    },
+    {
+      "epoch": 0.9284103051169417,
+      "grad_norm": 2.28125,
+      "learning_rate": 2.6850130350113303e-07,
+      "loss": 0.8222142219543457,
+      "step": 3910,
+      "token_acc": 0.7958430687655865
+    },
+    {
+      "epoch": 0.9307847560251692,
+      "grad_norm": 2.265625,
+      "learning_rate": 2.510877198506889e-07,
+      "loss": 0.7815386772155761,
+      "step": 3920,
+      "token_acc": 0.8082301335336742
+    },
+    {
+      "epoch": 0.9331592069333966,
+      "grad_norm": 2.53125,
+      "learning_rate": 2.3425072957221294e-07,
+      "loss": 0.7986605644226075,
+      "step": 3930,
+      "token_acc": 0.8075112548999469
+    },
+    {
+      "epoch": 0.9355336578416241,
+      "grad_norm": 2.296875,
+      "learning_rate": 2.179913284790558e-07,
+      "loss": 0.8034539222717285,
+      "step": 3940,
+      "token_acc": 0.8035468372249595
+    },
+    {
+      "epoch": 0.9379081087498516,
+      "grad_norm": 2.1875,
+      "learning_rate": 2.023104782234142e-07,
+      "loss": 0.8044528007507324,
+      "step": 3950,
+      "token_acc": 0.8022056750020959
+    },
+    {
+      "epoch": 0.940282559658079,
+      "grad_norm": 2.328125,
+      "learning_rate": 1.8720910623945431e-07,
+      "loss": 0.8020843505859375,
+      "step": 3960,
+      "token_acc": 0.8047856859372933
+    },
+    {
+      "epoch": 0.9426570105663066,
+      "grad_norm": 2.34375,
+      "learning_rate": 1.7268810568845552e-07,
+      "loss": 0.8500131607055664,
+      "step": 3970,
+      "token_acc": 0.7969773455767355
+    },
+    {
+      "epoch": 0.945031461474534,
+      "grad_norm": 2.203125,
+      "learning_rate": 1.587483354059949e-07,
+      "loss": 0.7900126934051513,
+      "step": 3980,
+      "token_acc": 0.8072901105467583
+    },
+    {
+      "epoch": 0.9474059123827615,
+      "grad_norm": 2.390625,
+      "learning_rate": 1.453906198511401e-07,
+      "loss": 0.801048755645752,
+      "step": 3990,
+      "token_acc": 0.8054952022657141
+    },
+    {
+      "epoch": 0.9497803632909889,
+      "grad_norm": 2.328125,
+      "learning_rate": 1.3261574905769625e-07,
+      "loss": 0.8011016845703125,
+      "step": 4000,
+      "token_acc": 0.8072040294262517
+    },
+    {
+      "epoch": 0.9521548141992164,
+      "grad_norm": 2.453125,
+      "learning_rate": 1.204244785874753e-07,
+      "loss": 0.7899654388427735,
+      "step": 4010,
+      "token_acc": 0.8079000117910624
+    },
+    {
+      "epoch": 0.954529265107444,
+      "grad_norm": 2.421875,
+      "learning_rate": 1.0881752948560975e-07,
+      "loss": 0.8631426811218261,
+      "step": 4020,
+      "token_acc": 0.7958376117608755
+    },
+    {
+      "epoch": 0.9569037160156714,
+      "grad_norm": 2.28125,
+      "learning_rate": 9.779558823790669e-08,
+      "loss": 0.8261016845703125,
+      "step": 4030,
+      "token_acc": 0.7995307266275329
+    },
+    {
+      "epoch": 0.9592781669238989,
+      "grad_norm": 2.109375,
+      "learning_rate": 8.735930673024806e-08,
+      "loss": 0.8057336807250977,
+      "step": 4040,
+      "token_acc": 0.8055790714273857
+    },
+    {
+      "epoch": 0.9616526178321263,
+      "grad_norm": 2.15625,
+      "learning_rate": 7.75093022100315e-08,
+      "loss": 0.8179000854492188,
+      "step": 4050,
+      "token_acc": 0.8038629336270067
+    },
+    {
+      "epoch": 0.9640270687403538,
+      "grad_norm": 2.203125,
+      "learning_rate": 6.824615724966843e-08,
+      "loss": 0.8242937088012695,
+      "step": 4060,
+      "token_acc": 0.799587730134397
+    },
+    {
+      "epoch": 0.9664015196485812,
+      "grad_norm": 2.21875,
+      "learning_rate": 5.957041971212274e-08,
+      "loss": 0.7834344863891601,
+      "step": 4070,
+      "token_acc": 0.8102642901044868
+    },
+    {
+      "epoch": 0.9687759705568088,
+      "grad_norm": 2.109375,
+      "learning_rate": 5.1482602718512245e-08,
+      "loss": 0.7839698791503906,
+      "step": 4080,
+      "token_acc": 0.8093756722320586
+    },
+    {
+      "epoch": 0.9711504214650362,
+      "grad_norm": 2.140625,
+      "learning_rate": 4.398318461775741e-08,
+      "loss": 0.8353322982788086,
+      "step": 4090,
+      "token_acc": 0.800353659676565
+    },
+    {
+      "epoch": 0.9735248723732637,
+      "grad_norm": 2.359375,
+      "learning_rate": 3.707260895829179e-08,
+      "loss": 0.7783904075622559,
+      "step": 4100,
+      "token_acc": 0.8112859459280585
+    },
+    {
+      "epoch": 0.9758993232814912,
+      "grad_norm": 2.28125,
+      "learning_rate": 3.0751284461828515e-08,
+      "loss": 0.7601756095886231,
+      "step": 4110,
+      "token_acc": 0.8119900190152202
+    },
+    {
+      "epoch": 0.9782737741897186,
+      "grad_norm": 2.4375,
+      "learning_rate": 2.501958499918522e-08,
+      "loss": 0.780480146408081,
+      "step": 4120,
+      "token_acc": 0.8106099356738743
+    },
+    {
+      "epoch": 0.9806482250979461,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.9877849568172846e-08,
+      "loss": 0.7624724864959717,
+      "step": 4130,
+      "token_acc": 0.8135922003980417
+    },
+    {
+      "epoch": 0.9830226760061735,
+      "grad_norm": 1.953125,
+      "learning_rate": 1.5326382273544993e-08,
+      "loss": 0.7853314399719238,
+      "step": 4140,
+      "token_acc": 0.8096398385745831
+    },
+    {
+      "epoch": 0.9853971269144011,
+      "grad_norm": 2.265625,
+      "learning_rate": 1.1365452309012315e-08,
+      "loss": 0.7484091281890869,
+      "step": 4150,
+      "token_acc": 0.8169504332867088
+    },
+    {
+      "epoch": 0.9877715778226285,
+      "grad_norm": 2.140625,
+      "learning_rate": 7.995293941321924e-09,
+      "loss": 0.815186595916748,
+      "step": 4160,
+      "token_acc": 0.8054412418644137
+    },
+    {
+      "epoch": 0.990146028730856,
+      "grad_norm": 2.046875,
+      "learning_rate": 5.216106496400697e-09,
+      "loss": 0.7996592044830322,
+      "step": 4170,
+      "token_acc": 0.805013573125212
+    },
+    {
+      "epoch": 0.9925204796390834,
+      "grad_norm": 2.453125,
+      "learning_rate": 3.028054347566922e-09,
+      "loss": 0.8040179252624512,
+      "step": 4180,
+      "token_acc": 0.805358027453299
+    },
+    {
+      "epoch": 0.9948949305473109,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.431266905808082e-09,
+      "loss": 0.8086426734924317,
+      "step": 4190,
+      "token_acc": 0.80584264244957
+    },
+    {
+      "epoch": 0.9972693814555385,
+      "grad_norm": 1.984375,
+      "learning_rate": 4.258386121280822e-10,
+      "loss": 0.8071908950805664,
+      "step": 4200,
+      "token_acc": 0.8064324873721819
+    },
+    {
+      "epoch": 0.9996438323637659,
+      "grad_norm": 2.515625,
+      "learning_rate": 1.1828931958390854e-11,
+      "loss": 0.7795707225799561,
+      "step": 4210,
+      "token_acc": 0.8079785327966532
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 4212,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4.9021915690644275e+17,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

models/sft/sft_f3f4_instruct/args.json

@@ -0,0 +1,344 @@
+{
+  "output_dir": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3F4_clean_1epoch_1gpu/v0-20260506-103602",
+  "per_device_train_batch_size": 4,
+  "num_train_epochs": 1.0,
+  "max_steps": -1,
+  "learning_rate": 2e-05,
+  "lr_scheduler_type": "cosine",
+  "lr_scheduler_kwargs": null,
+  "warmup_steps": 0,
+  "optim": "adamw_torch_fused",
+  "optim_args": null,
+  "weight_decay": 0.1,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.95,
+  "adam_epsilon": 1e-08,
+  "optim_target_modules": null,
+  "gradient_accumulation_steps": 8,
+  "average_tokens_across_devices": true,
+  "max_grad_norm": 1.0,
+  "label_smoothing_factor": 0.0,
+  "bf16": true,
+  "fp16": false,
+  "bf16_full_eval": false,
+  "fp16_full_eval": false,
+  "tf32": null,
+  "gradient_checkpointing": true,
+  "gradient_checkpointing_kwargs": null,
+  "torch_compile": false,
+  "torch_compile_backend": null,
+  "torch_compile_mode": null,
+  "use_liger_kernel": false,
+  "liger_kernel_config": null,
+  "use_cache": false,
+  "neftune_noise_alpha": null,
+  "torch_empty_cache_steps": null,
+  "auto_find_batch_size": false,
+  "logging_strategy": "steps",
+  "logging_steps": 10,
+  "logging_first_step": true,
+  "log_on_each_node": true,
+  "logging_nan_inf_filter": true,
+  "include_num_input_tokens_seen": false,
+  "log_level": "passive",
+  "log_level_replica": "warning",
+  "disable_tqdm": null,
+  "report_to": [
+    "tensorboard"
+  ],
+  "run_name": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3F4_clean_1epoch_1gpu/v0-20260506-103602",
+  "project": "huggingface",
+  "trackio_space_id": null,
+  "trackio_bucket_id": null,
+  "trackio_static_space_id": null,
+  "eval_strategy": "no",
+  "eval_steps": 500.0,
+  "eval_delay": 0,
+  "per_device_eval_batch_size": 1,
+  "prediction_loss_only": false,
+  "eval_on_start": false,
+  "eval_do_concat_batches": true,
+  "eval_use_gather_object": false,
+  "eval_accumulation_steps": null,
+  "include_for_metrics": [],
+  "batch_eval_metrics": false,
+  "save_only_model": false,
+  "save_strategy": "steps",
+  "save_steps": 500.0,
+  "save_on_each_node": false,
+  "save_total_limit": 5,
+  "enable_jit_checkpoint": false,
+  "push_to_hub": false,
+  "hub_token": null,
+  "hub_private_repo": null,
+  "hub_model_id": null,
+  "hub_strategy": "every_save",
+  "hub_always_push": false,
+  "hub_revision": null,
+  "load_best_model_at_end": false,
+  "metric_for_best_model": "loss",
+  "greater_is_better": false,
+  "ignore_data_skip": false,
+  "restore_callback_states_from_checkpoint": false,
+  "full_determinism": false,
+  "seed": 42,
+  "data_seed": 42,
+  "use_cpu": false,
+  "accelerator_config": {
+    "dispatch_batches": false
+  },
+  "parallelism_config": null,
+  "dataloader_drop_last": false,
+  "dataloader_num_workers": 8,
+  "dataloader_pin_memory": true,
+  "dataloader_persistent_workers": false,
+  "dataloader_prefetch_factor": null,
+  "remove_unused_columns": true,
+  "label_names": null,
+  "train_sampling_strategy": "random",
+  "length_column_name": "length",
+  "ddp_find_unused_parameters": null,
+  "ddp_bucket_cap_mb": null,
+  "ddp_broadcast_buffers": null,
+  "ddp_static_graph": null,
+  "ddp_backend": null,
+  "ddp_timeout": 18000000,
+  "fsdp": [],
+  "fsdp_config": null,
+  "deepspeed": null,
+  "debug": null,
+  "skip_memory_metrics": true,
+  "do_train": false,
+  "do_eval": false,
+  "do_predict": false,
+  "resume_from_checkpoint": null,
+  "warmup_ratio": 0.03,
+  "logging_dir": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3F4_clean_1epoch_1gpu/v0-20260506-103602/runs",
+  "local_rank": 0,
+  "sortish_sampler": false,
+  "predict_with_generate": false,
+  "generation_max_length": null,
+  "generation_num_beams": null,
+  "generation_config": null,
+  "tuner_backend": "peft",
+  "vit_gradient_checkpointing": false,
+  "router_aux_loss_coef": 0.0,
+  "enable_dft_loss": false,
+  "enable_channel_loss": false,
+  "safe_serialization": true,
+  "max_shard_size": "5GB",
+  "check_model": true,
+  "acc_strategy": "token",
+  "train_dataloader_shuffle": true,
+  "group_by_length": false,
+  "max_epochs": null,
+  "aligner_lr": null,
+  "vit_lr": null,
+  "use_logits_to_keep": null,
+  "ds3_gather_for_generation": true,
+  "resume_only_model": false,
+  "optimizer": null,
+  "loss_type": null,
+  "eval_metric": null,
+  "callbacks": [],
+  "early_stop_interval": null,
+  "eval_use_evalscope": false,
+  "eval_dataset": [],
+  "eval_dataset_args": null,
+  "eval_limit": null,
+  "eval_generation_config": null,
+  "extra_eval_args": null,
+  "tuner_type": "full",
+  "use_galore": false,
+  "galore_target_modules": null,
+  "galore_rank": 128,
+  "galore_update_proj_gap": 50,
+  "galore_scale": 1.0,
+  "galore_proj_type": "std",
+  "galore_optim_per_parameter": false,
+  "galore_with_embedding": false,
+  "galore_quantization": false,
+  "galore_proj_quant": false,
+  "galore_proj_bits": 4,
+  "galore_proj_group_size": 256,
+  "galore_cos_threshold": 0.4,
+  "galore_gamma_proj": 2,
+  "galore_queue_size": 5,
+  "lisa_activated_layers": 0,
+  "lisa_step_interval": 20,
+  "use_flash_ckpt": false,
+  "use_ray": false,
+  "ray_exp_name": null,
+  "device_groups": null,
+  "model": "Qwen/Qwen2.5-Coder-0.5B-Instruct",
+  "model_type": "qwen2",
+  "model_revision": null,
+  "task_type": "causal_lm",
+  "torch_dtype": "bfloat16",
+  "attn_impl": "flash_attn",
+  "experts_impl": null,
+  "new_special_tokens": [],
+  "num_labels": null,
+  "problem_type": null,
+  "rope_scaling": null,
+  "device_map": null,
+  "max_memory": {},
+  "max_model_len": null,
+  "local_repo_path": null,
+  "init_strategy": null,
+  "template": "qwen2_5",
+  "system": null,
+  "max_length": 4096,
+  "truncation_strategy": "delete",
+  "max_pixels": null,
+  "agent_template": null,
+  "norm_bbox": null,
+  "use_chat_template": true,
+  "padding_side": "right",
+  "padding_free": false,
+  "loss_scale": "default",
+  "sequence_parallel_size": 1,
+  "template_backend": "swift",
+  "response_prefix": null,
+  "enable_thinking": null,
+  "add_non_thinking_prefix": true,
+  "dataset": [
+    "/raid/data/weifeng/Datasets/Instruction-turning/scicode_sft_swift_clean.jsonl"
+  ],
+  "val_dataset": [],
+  "cached_dataset": [],
+  "cached_val_dataset": [],
+  "split_dataset_ratio": 0.0,
+  "dataset_num_proc": 8,
+  "load_from_cache_file": false,
+  "dataset_shuffle": true,
+  "val_dataset_shuffle": false,
+  "streaming": false,
+  "interleave_prob": null,
+  "stopping_strategy": "first_exhausted",
+  "shuffle_buffer_size": 1000,
+  "download_mode": "reuse_dataset_if_exists",
+  "columns": {},
+  "strict": false,
+  "model_name": null,
+  "model_author": null,
+  "custom_dataset_info": [],
+  "quant_method": null,
+  "quant_bits": null,
+  "hqq_axis": null,
+  "bnb_4bit_compute_dtype": "bfloat16",
+  "bnb_4bit_quant_type": "nf4",
+  "bnb_4bit_use_double_quant": true,
+  "bnb_4bit_quant_storage": null,
+  "max_new_tokens": 64,
+  "temperature": 0.0,
+  "top_k": null,
+  "top_p": null,
+  "repetition_penalty": null,
+  "num_beams": 1,
+  "stream": false,
+  "stop_words": [],
+  "logprobs": false,
+  "top_logprobs": null,
+  "structured_outputs_regex": null,
+  "adapters": [],
+  "external_plugins": [],
+  "custom_register_path": [],
+  "model_kwargs": {},
+  "load_args": false,
+  "load_data_args": false,
+  "packing": false,
+  "packing_length": null,
+  "packing_num_proc": 1,
+  "lazy_tokenize": false,
+  "use_hf": true,
+  "ignore_args_error": false,
+  "use_swift_lora": false,
+  "freeze_parameters": [],
+  "freeze_parameters_regex": null,
+  "freeze_parameters_ratio": 0.0,
+  "trainable_parameters": [],
+  "trainable_parameters_regex": null,
+  "freeze_llm": false,
+  "freeze_vit": true,
+  "freeze_aligner": true,
+  "target_modules": [
+    "all-linear"
+  ],
+  "target_regex": null,
+  "target_parameters": null,
+  "modules_to_save": [],
+  "lora_rank": 8,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_bias": "none",
+  "lora_dtype": null,
+  "lorap_lr_ratio": null,
+  "use_rslora": false,
+  "use_dora": false,
+  "lora_ga_batch_size": 2,
+  "lora_ga_iters": 2,
+  "lora_ga_max_length": 1024,
+  "lora_ga_direction": "ArB2r",
+  "lora_ga_scale": "stable",
+  "lora_ga_stable_gamma": 16,
+  "init_weights": true,
+  "fourier_n_frequency": 2000,
+  "fourier_scaling": 300.0,
+  "boft_block_size": 4,
+  "boft_block_num": 0,
+  "boft_n_butterfly_factor": 1,
+  "boft_dropout": 0.0,
+  "vera_rank": 256,
+  "vera_projection_prng_key": 0,
+  "vera_dropout": 0.0,
+  "vera_d_initial": 0.1,
+  "adapter_act": "gelu",
+  "adapter_length": 128,
+  "adalora_target_r": 8,
+  "adalora_init_r": 12,
+  "adalora_tinit": 0,
+  "adalora_tfinal": 0,
+  "adalora_deltaT": 1,
+  "adalora_beta1": 0.85,
+  "adalora_beta2": 0.85,
+  "adalora_orth_reg_weight": 0.5,
+  "llamapro_num_new_blocks": 4,
+  "llamapro_num_groups": null,
+  "reft_layer_key": null,
+  "reft_layers": null,
+  "reft_rank": 4,
+  "reft_intervention_type": "LoreftIntervention",
+  "reft_args": null,
+  "swanlab_token": null,
+  "swanlab_project": "ms-swift",
+  "swanlab_workspace": null,
+  "swanlab_exp_name": null,
+  "swanlab_notification_method": null,
+  "swanlab_webhook_url": null,
+  "swanlab_secret": null,
+  "swanlab_sender_email": null,
+  "swanlab_receiver_email": null,
+  "swanlab_smtp_server": null,
+  "swanlab_smtp_port": null,
+  "swanlab_email_language": "zh",
+  "swanlab_mode": "cloud",
+  "add_version": true,
+  "create_checkpoint_symlink": false,
+  "zero_hpz_partition_size": null,
+  "deepspeed_autotp_size": null,
+  "swift_version": "4.1.3",
+  "ckpt_dir": null,
+  "rank": 0,
+  "global_world_size": 1,
+  "local_world_size": 1,
+  "model_suffix": "Qwen2.5-Coder-0.5B-Instruct",
+  "model_info": "ModelInfo(model_type='qwen2', model_dir='/home/weifeng/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-0.5B-Instruct/snapshots/ea3f2471cf1b1f0db85067f1ef93848e38e88c25', torch_dtype=torch.bfloat16, max_model_len=32768, quant_method=None, quant_bits=None, rope_scaling={'rope_theta': 1000000.0, 'rope_type': 'default'}, is_moe_model=False, is_multimodal=False, config=None, task_type='causal_lm', num_labels=None)",
+  "model_meta": "ModelMeta(model_type='qwen2', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen1.5-0.5B-Chat', hf_model_id='Qwen/Qwen1.5-0.5B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-1.8B-Chat', hf_model_id='Qwen/Qwen1.5-1.8B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-4B-Chat', hf_model_id='Qwen/Qwen1.5-4B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-7B-Chat', hf_model_id='Qwen/Qwen1.5-7B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-14B-Chat', hf_model_id='Qwen/Qwen1.5-14B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-32B-Chat', hf_model_id='Qwen/Qwen1.5-32B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-72B-Chat', hf_model_id='Qwen/Qwen1.5-72B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-110B-Chat', hf_model_id='Qwen/Qwen1.5-110B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-0.5B', hf_model_id='Qwen/Qwen1.5-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-1.8B', hf_model_id='Qwen/Qwen1.5-1.8B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-4B', hf_model_id='Qwen/Qwen1.5-4B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-7B', hf_model_id='Qwen/Qwen1.5-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-14B', hf_model_id='Qwen/Qwen1.5-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-32B', hf_model_id='Qwen/Qwen1.5-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-72B', hf_model_id='Qwen/Qwen1.5-72B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-110B', hf_model_id='Qwen/Qwen1.5-110B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-4B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-4B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-7B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-7B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-14B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-14B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-32B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-32B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-72B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-72B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-110B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-110B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-4B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-4B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-7B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-7B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-14B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-14B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-72B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-72B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-0.5B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-0.5B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-1.8B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-1.8B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-4B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-4B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-7B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-7B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-14B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-14B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-32B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-32B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-72B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-72B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-110B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-110B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/CodeQwen1.5-7B', hf_model_id='Qwen/CodeQwen1.5-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/CodeQwen1.5-7B-Chat', hf_model_id='Qwen/CodeQwen1.5-7B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/CodeQwen1.5-7B-Chat-AWQ', hf_model_id='Qwen/CodeQwen1.5-7B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=['coding']), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2-0.5B-Instruct', hf_model_id='Qwen/Qwen2-0.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-1.5B-Instruct', hf_model_id='Qwen/Qwen2-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-7B-Instruct', hf_model_id='Qwen/Qwen2-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-72B-Instruct', hf_model_id='Qwen/Qwen2-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-0.5B', hf_model_id='Qwen/Qwen2-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-1.5B', hf_model_id='Qwen/Qwen2-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-7B', hf_model_id='Qwen/Qwen2-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-72B', hf_model_id='Qwen/Qwen2-72B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-0.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2-0.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-7B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2-7B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-72B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2-72B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-0.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2-0.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-1.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2-1.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-7B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2-7B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-72B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2-72B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-0.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2-0.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-1.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2-1.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-72B-Instruct-AWQ', hf_model_id='Qwen/Qwen2-72B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2-Math-1.5B-Instruct', hf_model_id='Qwen/Qwen2-Math-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-Math-7B-Instruct', hf_model_id='Qwen/Qwen2-Math-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-Math-72B-Instruct', hf_model_id='Qwen/Qwen2-Math-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-Math-1.5B', hf_model_id='Qwen/Qwen2-Math-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-Math-7B', hf_model_id='Qwen/Qwen2-Math-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-Math-72B', hf_model_id='Qwen/Qwen2-Math-72B', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=['math']), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-1M', hf_model_id='Qwen/Qwen2.5-7B-Instruct-1M', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-1M', hf_model_id='Qwen/Qwen2.5-14B-Instruct-1M', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='PowerInfer/SmallThinker-3B-Preview', hf_model_id='PowerInfer/SmallThinker-3B-Preview', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct', hf_model_id='Qwen/Qwen2.5-3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct', hf_model_id='Qwen/Qwen2.5-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct', hf_model_id='Qwen/Qwen2.5-14B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct', hf_model_id='Qwen/Qwen2.5-32B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct', hf_model_id='Qwen/Qwen2.5-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B', hf_model_id='Qwen/Qwen2.5-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B', hf_model_id='Qwen/Qwen2.5-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B', hf_model_id='Qwen/Qwen2.5-3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B', hf_model_id='Qwen/Qwen2.5-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B', hf_model_id='Qwen/Qwen2.5-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B', hf_model_id='Qwen/Qwen2.5-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B', hf_model_id='Qwen/Qwen2.5-72B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-3B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-14B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-32B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-72B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwen2_5', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B', hf_model_id='Qwen/Qwen2.5-Coder-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B', hf_model_id='Qwen/Qwen2.5-Coder-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B', hf_model_id='Qwen/Qwen2.5-Coder-3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B', hf_model_id='Qwen/Qwen2.5-Coder-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B', hf_model_id='Qwen/Qwen2.5-Coder-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B', hf_model_id='Qwen/Qwen2.5-Coder-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None)], template='qwen2_5', ignore_patterns=None, requires=None, tags=['coding']), ModelGroup(models=[Model(ms_model_id='moonshotai/Kimi-Dev-72B', hf_model_id='moonshotai/Kimi-Dev-72B', model_path=None, ms_revision=None, hf_revision=None)], template='qwen2_5', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-Math-1.5B-Instruct', hf_model_id='Qwen/Qwen2.5-Math-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Math-7B-Instruct', hf_model_id='Qwen/Qwen2.5-Math-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Math-72B-Instruct', hf_model_id='Qwen/Qwen2.5-Math-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Math-1.5B', hf_model_id='Qwen/Qwen2.5-Math-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Math-7B', hf_model_id='Qwen/Qwen2.5-Math-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Math-72B', hf_model_id='Qwen/Qwen2.5-Math-72B', model_path=None, ms_revision=None, hf_revision=None)], template='qwen2_5_math', ignore_patterns=None, requires=None, tags=['math']), ModelGroup(models=[Model(ms_model_id='AIDC-AI/Marco-o1', hf_model_id='AIDC-AI/Marco-o1', model_path=None, ms_revision=None, hf_revision=None)], template='marco_o1', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/QwQ-32B-Preview', hf_model_id='Qwen/QwQ-32B-Preview', model_path=None, ms_revision=None, hf_revision=None)], template='qwq_preview', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/QwQ-32B', hf_model_id='Qwen/QwQ-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/QwQ-32B-AWQ', hf_model_id='Qwen/QwQ-32B-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwq', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B', hf_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-7B', hf_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-14B', hf_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-32B', hf_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='iic/QwenLong-L1-32B', hf_model_id='Tongyi-Zhiwen/QwenLong-L1-32B', model_path=None, ms_revision=None, hf_revision=None)], template='deepseek_r1', ignore_patterns=None, requires=None, tags=[])], loader=<class 'swift.model.register.ModelLoader'>, template='qwen2_5', model_arch=ModelKeys(arch_name='llama', embedding='model.embed_tokens', module_list='model.layers', lm_head='lm_head', q_proj='model.layers.{}.self_attn.q_proj', k_proj='model.layers.{}.self_attn.k_proj', v_proj='model.layers.{}.self_attn.v_proj', o_proj='model.layers.{}.self_attn.o_proj', attention='model.layers.{}.self_attn', mlp='model.layers.{}.mlp', down_proj='model.layers.{}.mlp.down_proj', qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None), mcore_model_type=None, architectures=['Qwen2ForCausalLM'], additional_saved_files=[], torch_dtype=None, is_multimodal=False, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.37'], tags=['coding'])",
+  "model_dir": "/home/weifeng/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-0.5B-Instruct/snapshots/ea3f2471cf1b1f0db85067f1ef93848e38e88c25",
+  "template_meta": "Qwen2_5TemplateMeta(template_type='qwen2_5', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=<class 'swift.template.base.Template'>, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system='You are Qwen, created by Alibaba Cloud. You are a helpful assistant.', auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='', non_thinking_prefix='', history_thinking_prefix='')",
+  "_val_dataset_exists": false,
+  "hub": "<class 'swift.hub.hub.HFHub'>",
+  "evaluation_strategy": "steps",
+  "training_args": "Seq2SeqTrainingArguments(output_dir='/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3F4_clean_1epoch_1gpu/v0-20260506-103602', per_device_train_batch_size=4, num_train_epochs=1.0, max_steps=-1, learning_rate=2e-05, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, lr_scheduler_kwargs=None, warmup_steps=0.03, optim=<OptimizerNames.ADAMW_TORCH_FUSED: 'adamw_torch_fused'>, optim_args=None, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, optim_target_modules=None, gradient_accumulation_steps=8, average_tokens_across_devices=None, max_grad_norm=1.0, label_smoothing_factor=0.0, bf16=True, fp16=False, bf16_full_eval=False, fp16_full_eval=False, tf32=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, use_liger_kernel=False, liger_kernel_config=None, use_cache=False, neftune_noise_alpha=None, torch_empty_cache_steps=None, auto_find_batch_size=False, logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_steps=10, logging_first_step=True, log_on_each_node=True, logging_nan_inf_filter=True, include_num_input_tokens_seen=None, log_level='passive', log_level_replica='warning', disable_tqdm=False, report_to=['tensorboard'], run_name='/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3F4_clean_1epoch_1gpu/v0-20260506-103602', project='huggingface', trackio_space_id=None, trackio_bucket_id=None, trackio_static_space_id=None, eval_strategy=<IntervalStrategy.NO: 'no'>, eval_steps=500.0, eval_delay=0, per_device_eval_batch_size=1, prediction_loss_only=False, eval_on_start=False, eval_do_concat_batches=True, eval_use_gather_object=False, eval_accumulation_steps=None, include_for_metrics=[], batch_eval_metrics=False, save_only_model=False, save_strategy=<SaveStrategy.STEPS: 'steps'>, save_steps=500, save_on_each_node=False, save_total_limit=5, enable_jit_checkpoint=False, push_to_hub=False, hub_token=None, hub_private_repo=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_always_push=False, hub_revision=None, load_best_model_at_end=False, metric_for_best_model='loss', greater_is_better=False, ignore_data_skip=False, restore_callback_states_from_checkpoint=False, full_determinism=False, seed=42, data_seed=42, use_cpu=False, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), parallelism_config=None, dataloader_drop_last=False, dataloader_num_workers=8, dataloader_pin_memory=True, dataloader_persistent_workers=False, dataloader_prefetch_factor=2, remove_unused_columns=False, label_names=None, train_sampling_strategy='random', length_column_name='length', ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, ddp_static_graph=None, ddp_backend=None, ddp_timeout=18000000, fsdp=[], fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, deepspeed=None, debug=[], skip_memory_metrics=True, do_train=False, do_eval=False, do_predict=False, resume_from_checkpoint=None, warmup_ratio=0.03, logging_dir='/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3F4_clean_1epoch_1gpu/v0-20260506-103602/runs', local_rank=0, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, tuner_backend='peft', vit_gradient_checkpointing=False, router_aux_loss_coef=0.0, enable_dft_loss=False, enable_channel_loss=False, safe_serialization=True, max_shard_size='5GB', check_model=True, acc_strategy='token', train_dataloader_shuffle=True, group_by_length=False, max_epochs=None, aligner_lr=None, vit_lr=None, use_logits_to_keep=None, ds3_gather_for_generation=True, resume_only_model=False, optimizer=None, loss_type=None, eval_metric=None, callbacks=[], early_stop_interval=None, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, extra_eval_args=None, tuner_type='full', use_galore=False, galore_target_modules=None, galore_rank=128, galore_update_proj_gap=50, galore_scale=1.0, galore_proj_type='std', galore_optim_per_parameter=False, galore_with_embedding=False, galore_quantization=False, galore_proj_quant=False, galore_proj_bits=4, galore_proj_group_size=256, galore_cos_threshold=0.4, galore_gamma_proj=2, galore_queue_size=5, lisa_activated_layers=0, lisa_step_interval=20, use_flash_ckpt=False)"
+}

models/sft/sft_f3f4_instruct/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

models/sft/sft_f3f4_instruct/config.json

@@ -0,0 +1,57 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 896,
+  "initializer_range": 0.02,
+  "intermediate_size": 4864,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 24,
+  "model_type": "qwen2",
+  "num_attention_heads": 14,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 2,
+  "pad_token_id": 151643,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 1000000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.6.2",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

models/sft/sft_f3f4_instruct/generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.05,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "5.6.2"
+}

models/sft/sft_f3f4_instruct/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fc57c00fa0e875ea148c69d11a87a1af3f9b4b6571f2fdbeeaa17ba4220f373
+size 988097824

models/sft/sft_f3f4_instruct/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892

models/sft/sft_f3f4_instruct/tokenizer_config.json

@@ -0,0 +1,30 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

models/sft/sft_f4_instruct/args.json

@@ -0,0 +1,344 @@
+{
+  "output_dir": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F4_clean_1epoch_1gpu/v3-20260505-141059",
+  "per_device_train_batch_size": 4,
+  "num_train_epochs": 1.0,
+  "max_steps": -1,
+  "learning_rate": 2e-05,
+  "lr_scheduler_type": "cosine",
+  "lr_scheduler_kwargs": null,
+  "warmup_steps": 0,
+  "optim": "adamw_torch_fused",
+  "optim_args": null,
+  "weight_decay": 0.1,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.95,
+  "adam_epsilon": 1e-08,
+  "optim_target_modules": null,
+  "gradient_accumulation_steps": 8,
+  "average_tokens_across_devices": true,
+  "max_grad_norm": 1.0,
+  "label_smoothing_factor": 0.0,
+  "bf16": true,
+  "fp16": false,
+  "bf16_full_eval": false,
+  "fp16_full_eval": false,
+  "tf32": null,
+  "gradient_checkpointing": true,
+  "gradient_checkpointing_kwargs": null,
+  "torch_compile": false,
+  "torch_compile_backend": null,
+  "torch_compile_mode": null,
+  "use_liger_kernel": false,
+  "liger_kernel_config": null,
+  "use_cache": false,
+  "neftune_noise_alpha": null,
+  "torch_empty_cache_steps": null,
+  "auto_find_batch_size": false,
+  "logging_strategy": "steps",
+  "logging_steps": 10,
+  "logging_first_step": true,
+  "log_on_each_node": true,
+  "logging_nan_inf_filter": true,
+  "include_num_input_tokens_seen": false,
+  "log_level": "passive",
+  "log_level_replica": "warning",
+  "disable_tqdm": null,
+  "report_to": [
+    "tensorboard"
+  ],
+  "run_name": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F4_clean_1epoch_1gpu/v3-20260505-141059",
+  "project": "huggingface",
+  "trackio_space_id": null,
+  "trackio_bucket_id": null,
+  "trackio_static_space_id": null,
+  "eval_strategy": "no",
+  "eval_steps": 100.0,
+  "eval_delay": 0,
+  "per_device_eval_batch_size": 1,
+  "prediction_loss_only": false,
+  "eval_on_start": false,
+  "eval_do_concat_batches": true,
+  "eval_use_gather_object": false,
+  "eval_accumulation_steps": null,
+  "include_for_metrics": [],
+  "batch_eval_metrics": false,
+  "save_only_model": false,
+  "save_strategy": "steps",
+  "save_steps": 100.0,
+  "save_on_each_node": false,
+  "save_total_limit": 5,
+  "enable_jit_checkpoint": false,
+  "push_to_hub": false,
+  "hub_token": null,
+  "hub_private_repo": null,
+  "hub_model_id": null,
+  "hub_strategy": "every_save",
+  "hub_always_push": false,
+  "hub_revision": null,
+  "load_best_model_at_end": false,
+  "metric_for_best_model": "loss",
+  "greater_is_better": false,
+  "ignore_data_skip": false,
+  "restore_callback_states_from_checkpoint": false,
+  "full_determinism": false,
+  "seed": 42,
+  "data_seed": 42,
+  "use_cpu": false,
+  "accelerator_config": {
+    "dispatch_batches": false
+  },
+  "parallelism_config": null,
+  "dataloader_drop_last": false,
+  "dataloader_num_workers": 8,
+  "dataloader_pin_memory": true,
+  "dataloader_persistent_workers": false,
+  "dataloader_prefetch_factor": null,
+  "remove_unused_columns": true,
+  "label_names": null,
+  "train_sampling_strategy": "random",
+  "length_column_name": "length",
+  "ddp_find_unused_parameters": null,
+  "ddp_bucket_cap_mb": null,
+  "ddp_broadcast_buffers": null,
+  "ddp_static_graph": null,
+  "ddp_backend": null,
+  "ddp_timeout": 18000000,
+  "fsdp": [],
+  "fsdp_config": null,
+  "deepspeed": null,
+  "debug": null,
+  "skip_memory_metrics": true,
+  "do_train": false,
+  "do_eval": false,
+  "do_predict": false,
+  "resume_from_checkpoint": null,
+  "warmup_ratio": 0.03,
+  "logging_dir": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F4_clean_1epoch_1gpu/v3-20260505-141059/runs",
+  "local_rank": 0,
+  "sortish_sampler": false,
+  "predict_with_generate": false,
+  "generation_max_length": null,
+  "generation_num_beams": null,
+  "generation_config": null,
+  "tuner_backend": "peft",
+  "vit_gradient_checkpointing": false,
+  "router_aux_loss_coef": 0.0,
+  "enable_dft_loss": false,
+  "enable_channel_loss": false,
+  "safe_serialization": true,
+  "max_shard_size": "5GB",
+  "check_model": true,
+  "acc_strategy": "token",
+  "train_dataloader_shuffle": true,
+  "group_by_length": false,
+  "max_epochs": null,
+  "aligner_lr": null,
+  "vit_lr": null,
+  "use_logits_to_keep": null,
+  "ds3_gather_for_generation": true,
+  "resume_only_model": false,
+  "optimizer": null,
+  "loss_type": null,
+  "eval_metric": null,
+  "callbacks": [],
+  "early_stop_interval": null,
+  "eval_use_evalscope": false,
+  "eval_dataset": [],
+  "eval_dataset_args": null,
+  "eval_limit": null,
+  "eval_generation_config": null,
+  "extra_eval_args": null,
+  "tuner_type": "full",
+  "use_galore": false,
+  "galore_target_modules": null,
+  "galore_rank": 128,
+  "galore_update_proj_gap": 50,
+  "galore_scale": 1.0,
+  "galore_proj_type": "std",
+  "galore_optim_per_parameter": false,
+  "galore_with_embedding": false,
+  "galore_quantization": false,
+  "galore_proj_quant": false,
+  "galore_proj_bits": 4,
+  "galore_proj_group_size": 256,
+  "galore_cos_threshold": 0.4,
+  "galore_gamma_proj": 2,
+  "galore_queue_size": 5,
+  "lisa_activated_layers": 0,
+  "lisa_step_interval": 20,
+  "use_flash_ckpt": false,
+  "use_ray": false,
+  "ray_exp_name": null,
+  "device_groups": null,
+  "model": "Qwen/Qwen2.5-Coder-0.5B-Instruct",
+  "model_type": "qwen2",
+  "model_revision": null,
+  "task_type": "causal_lm",
+  "torch_dtype": "bfloat16",
+  "attn_impl": "sdpa",
+  "experts_impl": null,
+  "new_special_tokens": [],
+  "num_labels": null,
+  "problem_type": null,
+  "rope_scaling": null,
+  "device_map": null,
+  "max_memory": {},
+  "max_model_len": null,
+  "local_repo_path": null,
+  "init_strategy": null,
+  "template": "qwen2_5",
+  "system": null,
+  "max_length": 4096,
+  "truncation_strategy": "delete",
+  "max_pixels": null,
+  "agent_template": null,
+  "norm_bbox": null,
+  "use_chat_template": true,
+  "padding_side": "right",
+  "padding_free": false,
+  "loss_scale": "default",
+  "sequence_parallel_size": 1,
+  "template_backend": "swift",
+  "response_prefix": null,
+  "enable_thinking": null,
+  "add_non_thinking_prefix": true,
+  "dataset": [
+    "/raid/data/weifeng/Datasets/Instruction-turning/scicode_sft_swift_F4_clean.jsonl"
+  ],
+  "val_dataset": [],
+  "cached_dataset": [],
+  "cached_val_dataset": [],
+  "split_dataset_ratio": 0.0,
+  "dataset_num_proc": 8,
+  "load_from_cache_file": false,
+  "dataset_shuffle": true,
+  "val_dataset_shuffle": false,
+  "streaming": false,
+  "interleave_prob": null,
+  "stopping_strategy": "first_exhausted",
+  "shuffle_buffer_size": 1000,
+  "download_mode": "reuse_dataset_if_exists",
+  "columns": {},
+  "strict": false,
+  "model_name": null,
+  "model_author": null,
+  "custom_dataset_info": [],
+  "quant_method": null,
+  "quant_bits": null,
+  "hqq_axis": null,
+  "bnb_4bit_compute_dtype": "bfloat16",
+  "bnb_4bit_quant_type": "nf4",
+  "bnb_4bit_use_double_quant": true,
+  "bnb_4bit_quant_storage": null,
+  "max_new_tokens": 64,
+  "temperature": 0.0,
+  "top_k": null,
+  "top_p": null,
+  "repetition_penalty": null,
+  "num_beams": 1,
+  "stream": false,
+  "stop_words": [],
+  "logprobs": false,
+  "top_logprobs": null,
+  "structured_outputs_regex": null,
+  "adapters": [],
+  "external_plugins": [],
+  "custom_register_path": [],
+  "model_kwargs": {},
+  "load_args": false,
+  "load_data_args": false,
+  "packing": false,
+  "packing_length": null,
+  "packing_num_proc": 1,
+  "lazy_tokenize": false,
+  "use_hf": true,
+  "ignore_args_error": false,
+  "use_swift_lora": false,
+  "freeze_parameters": [],
+  "freeze_parameters_regex": null,
+  "freeze_parameters_ratio": 0.0,
+  "trainable_parameters": [],
+  "trainable_parameters_regex": null,
+  "freeze_llm": false,
+  "freeze_vit": true,
+  "freeze_aligner": true,
+  "target_modules": [
+    "all-linear"
+  ],
+  "target_regex": null,
+  "target_parameters": null,
+  "modules_to_save": [],
+  "lora_rank": 8,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_bias": "none",
+  "lora_dtype": null,
+  "lorap_lr_ratio": null,
+  "use_rslora": false,
+  "use_dora": false,
+  "lora_ga_batch_size": 2,
+  "lora_ga_iters": 2,
+  "lora_ga_max_length": 1024,
+  "lora_ga_direction": "ArB2r",
+  "lora_ga_scale": "stable",
+  "lora_ga_stable_gamma": 16,
+  "init_weights": true,
+  "fourier_n_frequency": 2000,
+  "fourier_scaling": 300.0,
+  "boft_block_size": 4,
+  "boft_block_num": 0,
+  "boft_n_butterfly_factor": 1,
+  "boft_dropout": 0.0,
+  "vera_rank": 256,
+  "vera_projection_prng_key": 0,
+  "vera_dropout": 0.0,
+  "vera_d_initial": 0.1,
+  "adapter_act": "gelu",
+  "adapter_length": 128,
+  "adalora_target_r": 8,
+  "adalora_init_r": 12,
+  "adalora_tinit": 0,
+  "adalora_tfinal": 0,
+  "adalora_deltaT": 1,
+  "adalora_beta1": 0.85,
+  "adalora_beta2": 0.85,
+  "adalora_orth_reg_weight": 0.5,
+  "llamapro_num_new_blocks": 4,
+  "llamapro_num_groups": null,
+  "reft_layer_key": null,
+  "reft_layers": null,
+  "reft_rank": 4,
+  "reft_intervention_type": "LoreftIntervention",
+  "reft_args": null,
+  "swanlab_token": null,
+  "swanlab_project": "ms-swift",
+  "swanlab_workspace": null,
+  "swanlab_exp_name": null,
+  "swanlab_notification_method": null,
+  "swanlab_webhook_url": null,
+  "swanlab_secret": null,
+  "swanlab_sender_email": null,
+  "swanlab_receiver_email": null,
+  "swanlab_smtp_server": null,
+  "swanlab_smtp_port": null,
+  "swanlab_email_language": "zh",
+  "swanlab_mode": "cloud",
+  "add_version": true,
+  "create_checkpoint_symlink": false,
+  "zero_hpz_partition_size": null,
+  "deepspeed_autotp_size": null,
+  "swift_version": "4.1.3",
+  "ckpt_dir": null,
+  "rank": 0,
+  "global_world_size": 1,
+  "local_world_size": 1,
+  "model_suffix": "Qwen2.5-Coder-0.5B-Instruct",
+  "model_info": "ModelInfo(model_type='qwen2', model_dir='/home/weifeng/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-0.5B-Instruct/snapshots/ea3f2471cf1b1f0db85067f1ef93848e38e88c25', torch_dtype=torch.bfloat16, max_model_len=32768, quant_method=None, quant_bits=None, rope_scaling={'rope_theta': 1000000.0, 'rope_type': 'default'}, is_moe_model=False, is_multimodal=False, config=None, task_type='causal_lm', num_labels=None)",
+  "model_meta": "ModelMeta(model_type='qwen2', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen1.5-0.5B-Chat', hf_model_id='Qwen/Qwen1.5-0.5B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-1.8B-Chat', hf_model_id='Qwen/Qwen1.5-1.8B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-4B-Chat', hf_model_id='Qwen/Qwen1.5-4B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-7B-Chat', hf_model_id='Qwen/Qwen1.5-7B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-14B-Chat', hf_model_id='Qwen/Qwen1.5-14B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-32B-Chat', hf_model_id='Qwen/Qwen1.5-32B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-72B-Chat', hf_model_id='Qwen/Qwen1.5-72B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-110B-Chat', hf_model_id='Qwen/Qwen1.5-110B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-0.5B', hf_model_id='Qwen/Qwen1.5-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-1.8B', hf_model_id='Qwen/Qwen1.5-1.8B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-4B', hf_model_id='Qwen/Qwen1.5-4B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-7B', hf_model_id='Qwen/Qwen1.5-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-14B', hf_model_id='Qwen/Qwen1.5-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-32B', hf_model_id='Qwen/Qwen1.5-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-72B', hf_model_id='Qwen/Qwen1.5-72B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-110B', hf_model_id='Qwen/Qwen1.5-110B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-4B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-4B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-7B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-7B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-14B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-14B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-32B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-32B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-72B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-72B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-110B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-110B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-4B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-4B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-7B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-7B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-14B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-14B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-72B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-72B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-0.5B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-0.5B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-1.8B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-1.8B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-4B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-4B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-7B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-7B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-14B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-14B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-32B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-32B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-72B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-72B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-110B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-110B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/CodeQwen1.5-7B', hf_model_id='Qwen/CodeQwen1.5-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/CodeQwen1.5-7B-Chat', hf_model_id='Qwen/CodeQwen1.5-7B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/CodeQwen1.5-7B-Chat-AWQ', hf_model_id='Qwen/CodeQwen1.5-7B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=['coding']), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2-0.5B-Instruct', hf_model_id='Qwen/Qwen2-0.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-1.5B-Instruct', hf_model_id='Qwen/Qwen2-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-7B-Instruct', hf_model_id='Qwen/Qwen2-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-72B-Instruct', hf_model_id='Qwen/Qwen2-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-0.5B', hf_model_id='Qwen/Qwen2-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-1.5B', hf_model_id='Qwen/Qwen2-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-7B', hf_model_id='Qwen/Qwen2-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-72B', hf_model_id='Qwen/Qwen2-72B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-0.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2-0.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-7B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2-7B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-72B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2-72B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-0.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2-0.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-1.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2-1.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-7B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2-7B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-72B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2-72B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-0.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2-0.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-1.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2-1.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-72B-Instruct-AWQ', hf_model_id='Qwen/Qwen2-72B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2-Math-1.5B-Instruct', hf_model_id='Qwen/Qwen2-Math-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-Math-7B-Instruct', hf_model_id='Qwen/Qwen2-Math-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-Math-72B-Instruct', hf_model_id='Qwen/Qwen2-Math-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-Math-1.5B', hf_model_id='Qwen/Qwen2-Math-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-Math-7B', hf_model_id='Qwen/Qwen2-Math-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-Math-72B', hf_model_id='Qwen/Qwen2-Math-72B', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=['math']), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-1M', hf_model_id='Qwen/Qwen2.5-7B-Instruct-1M', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-1M', hf_model_id='Qwen/Qwen2.5-14B-Instruct-1M', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='PowerInfer/SmallThinker-3B-Preview', hf_model_id='PowerInfer/SmallThinker-3B-Preview', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct', hf_model_id='Qwen/Qwen2.5-3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct', hf_model_id='Qwen/Qwen2.5-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct', hf_model_id='Qwen/Qwen2.5-14B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct', hf_model_id='Qwen/Qwen2.5-32B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct', hf_model_id='Qwen/Qwen2.5-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B', hf_model_id='Qwen/Qwen2.5-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B', hf_model_id='Qwen/Qwen2.5-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B', hf_model_id='Qwen/Qwen2.5-3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B', hf_model_id='Qwen/Qwen2.5-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B', hf_model_id='Qwen/Qwen2.5-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B', hf_model_id='Qwen/Qwen2.5-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B', hf_model_id='Qwen/Qwen2.5-72B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-3B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-14B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-32B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-72B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwen2_5', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B', hf_model_id='Qwen/Qwen2.5-Coder-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B', hf_model_id='Qwen/Qwen2.5-Coder-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B', hf_model_id='Qwen/Qwen2.5-Coder-3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B', hf_model_id='Qwen/Qwen2.5-Coder-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B', hf_model_id='Qwen/Qwen2.5-Coder-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B', hf_model_id='Qwen/Qwen2.5-Coder-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None)], template='qwen2_5', ignore_patterns=None, requires=None, tags=['coding']), ModelGroup(models=[Model(ms_model_id='moonshotai/Kimi-Dev-72B', hf_model_id='moonshotai/Kimi-Dev-72B', model_path=None, ms_revision=None, hf_revision=None)], template='qwen2_5', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-Math-1.5B-Instruct', hf_model_id='Qwen/Qwen2.5-Math-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Math-7B-Instruct', hf_model_id='Qwen/Qwen2.5-Math-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Math-72B-Instruct', hf_model_id='Qwen/Qwen2.5-Math-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Math-1.5B', hf_model_id='Qwen/Qwen2.5-Math-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Math-7B', hf_model_id='Qwen/Qwen2.5-Math-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Math-72B', hf_model_id='Qwen/Qwen2.5-Math-72B', model_path=None, ms_revision=None, hf_revision=None)], template='qwen2_5_math', ignore_patterns=None, requires=None, tags=['math']), ModelGroup(models=[Model(ms_model_id='AIDC-AI/Marco-o1', hf_model_id='AIDC-AI/Marco-o1', model_path=None, ms_revision=None, hf_revision=None)], template='marco_o1', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/QwQ-32B-Preview', hf_model_id='Qwen/QwQ-32B-Preview', model_path=None, ms_revision=None, hf_revision=None)], template='qwq_preview', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/QwQ-32B', hf_model_id='Qwen/QwQ-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/QwQ-32B-AWQ', hf_model_id='Qwen/QwQ-32B-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwq', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B', hf_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-7B', hf_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-14B', hf_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-32B', hf_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='iic/QwenLong-L1-32B', hf_model_id='Tongyi-Zhiwen/QwenLong-L1-32B', model_path=None, ms_revision=None, hf_revision=None)], template='deepseek_r1', ignore_patterns=None, requires=None, tags=[])], loader=<class 'swift.model.register.ModelLoader'>, template='qwen2_5', model_arch=ModelKeys(arch_name='llama', embedding='model.embed_tokens', module_list='model.layers', lm_head='lm_head', q_proj='model.layers.{}.self_attn.q_proj', k_proj='model.layers.{}.self_attn.k_proj', v_proj='model.layers.{}.self_attn.v_proj', o_proj='model.layers.{}.self_attn.o_proj', attention='model.layers.{}.self_attn', mlp='model.layers.{}.mlp', down_proj='model.layers.{}.mlp.down_proj', qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None), mcore_model_type=None, architectures=['Qwen2ForCausalLM'], additional_saved_files=[], torch_dtype=None, is_multimodal=False, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.37'], tags=['coding'])",
+  "model_dir": "/home/weifeng/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-0.5B-Instruct/snapshots/ea3f2471cf1b1f0db85067f1ef93848e38e88c25",
+  "template_meta": "Qwen2_5TemplateMeta(template_type='qwen2_5', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=<class 'swift.template.base.Template'>, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system='You are Qwen, created by Alibaba Cloud. You are a helpful assistant.', auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='', non_thinking_prefix='', history_thinking_prefix='')",
+  "_val_dataset_exists": false,
+  "hub": "<class 'swift.hub.hub.HFHub'>",
+  "evaluation_strategy": "steps",
+  "training_args": "Seq2SeqTrainingArguments(output_dir='/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F4_clean_1epoch_1gpu/v3-20260505-141059', per_device_train_batch_size=4, num_train_epochs=1.0, max_steps=-1, learning_rate=2e-05, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, lr_scheduler_kwargs=None, warmup_steps=0.03, optim=<OptimizerNames.ADAMW_TORCH_FUSED: 'adamw_torch_fused'>, optim_args=None, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, optim_target_modules=None, gradient_accumulation_steps=8, average_tokens_across_devices=None, max_grad_norm=1.0, label_smoothing_factor=0.0, bf16=True, fp16=False, bf16_full_eval=False, fp16_full_eval=False, tf32=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, use_liger_kernel=False, liger_kernel_config=None, use_cache=False, neftune_noise_alpha=None, torch_empty_cache_steps=None, auto_find_batch_size=False, logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_steps=10, logging_first_step=True, log_on_each_node=True, logging_nan_inf_filter=True, include_num_input_tokens_seen=None, log_level='passive', log_level_replica='warning', disable_tqdm=False, report_to=['tensorboard'], run_name='/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F4_clean_1epoch_1gpu/v3-20260505-141059', project='huggingface', trackio_space_id=None, trackio_bucket_id=None, trackio_static_space_id=None, eval_strategy=<IntervalStrategy.NO: 'no'>, eval_steps=100.0, eval_delay=0, per_device_eval_batch_size=1, prediction_loss_only=False, eval_on_start=False, eval_do_concat_batches=True, eval_use_gather_object=False, eval_accumulation_steps=None, include_for_metrics=[], batch_eval_metrics=False, save_only_model=False, save_strategy=<SaveStrategy.STEPS: 'steps'>, save_steps=100, save_on_each_node=False, save_total_limit=5, enable_jit_checkpoint=False, push_to_hub=False, hub_token=None, hub_private_repo=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_always_push=False, hub_revision=None, load_best_model_at_end=False, metric_for_best_model='loss', greater_is_better=False, ignore_data_skip=False, restore_callback_states_from_checkpoint=False, full_determinism=False, seed=42, data_seed=42, use_cpu=False, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), parallelism_config=None, dataloader_drop_last=False, dataloader_num_workers=8, dataloader_pin_memory=True, dataloader_persistent_workers=False, dataloader_prefetch_factor=2, remove_unused_columns=False, label_names=None, train_sampling_strategy='random', length_column_name='length', ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, ddp_static_graph=None, ddp_backend=None, ddp_timeout=18000000, fsdp=[], fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, deepspeed=None, debug=[], skip_memory_metrics=True, do_train=False, do_eval=False, do_predict=False, resume_from_checkpoint=None, warmup_ratio=0.03, logging_dir='/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F4_clean_1epoch_1gpu/v3-20260505-141059/runs', local_rank=0, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, tuner_backend='peft', vit_gradient_checkpointing=False, router_aux_loss_coef=0.0, enable_dft_loss=False, enable_channel_loss=False, safe_serialization=True, max_shard_size='5GB', check_model=True, acc_strategy='token', train_dataloader_shuffle=True, group_by_length=False, max_epochs=None, aligner_lr=None, vit_lr=None, use_logits_to_keep=None, ds3_gather_for_generation=True, resume_only_model=False, optimizer=None, loss_type=None, eval_metric=None, callbacks=[], early_stop_interval=None, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, extra_eval_args=None, tuner_type='full', use_galore=False, galore_target_modules=None, galore_rank=128, galore_update_proj_gap=50, galore_scale=1.0, galore_proj_type='std', galore_optim_per_parameter=False, galore_with_embedding=False, galore_quantization=False, galore_proj_quant=False, galore_proj_bits=4, galore_proj_group_size=256, galore_cos_threshold=0.4, galore_gamma_proj=2, galore_queue_size=5, lisa_activated_layers=0, lisa_step_interval=20, use_flash_ckpt=False)"
+}

models/sft/sft_f4_instruct/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

models/sft/sft_f4_instruct/config.json

@@ -0,0 +1,57 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 896,
+  "initializer_range": 0.02,
+  "intermediate_size": 4864,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 24,
+  "model_type": "qwen2",
+  "num_attention_heads": 14,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 2,
+  "pad_token_id": 151643,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 1000000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.6.2",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

models/sft/sft_f4_instruct/generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.05,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "5.6.2"
+}

models/sft/sft_f4_instruct/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9a6cc732c33b8ee955f2766c7be98a91272603d1be38f51569bf13e1ac292b1
+size 988097824

models/sft/sft_f4_instruct/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892

models/sft/sft_f4_instruct/tokenizer_config.json

@@ -0,0 +1,30 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

models/sft/sft_f4_instruct/trainer_state.json

@@ -0,0 +1,426 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 100.0,
+  "global_step": 484,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.002067717756526234,
+      "grad_norm": 4.46875,
+      "learning_rate": 1.3333333333333334e-06,
+      "loss": 0.8509945869445801,
+      "step": 1,
+      "token_acc": 0.7853175561458585
+    },
+    {
+      "epoch": 0.020677177565262342,
+      "grad_norm": 1.828125,
+      "learning_rate": 1.3333333333333333e-05,
+      "loss": 0.8491229481167264,
+      "step": 10,
+      "token_acc": 0.78339043091193
+    },
+    {
+      "epoch": 0.041354355130524684,
+      "grad_norm": 1.4296875,
+      "learning_rate": 1.9994391800229666e-05,
+      "loss": 0.7549973011016846,
+      "step": 20,
+      "token_acc": 0.7997072436054726
+    },
+    {
+      "epoch": 0.062031532695787026,
+      "grad_norm": 1.25,
+      "learning_rate": 1.9949563937297045e-05,
+      "loss": 0.734230375289917,
+      "step": 30,
+      "token_acc": 0.8047420407633951
+    },
+    {
+      "epoch": 0.08270871026104937,
+      "grad_norm": 1.2421875,
+      "learning_rate": 1.9860109277923417e-05,
+      "loss": 0.7340453624725342,
+      "step": 40,
+      "token_acc": 0.8038525401200097
+    },
+    {
+      "epoch": 0.10338588782631171,
+      "grad_norm": 1.1953125,
+      "learning_rate": 1.972642905324813e-05,
+      "loss": 0.6950289249420166,
+      "step": 50,
+      "token_acc": 0.8114663162785941
+    },
+    {
+      "epoch": 0.12406306539157405,
+      "grad_norm": 1.0703125,
+      "learning_rate": 1.9549122859415538e-05,
+      "loss": 0.6816429138183594,
+      "step": 60,
+      "token_acc": 0.8157978734876097
+    },
+    {
+      "epoch": 0.14474024295683638,
+      "grad_norm": 1.1953125,
+      "learning_rate": 1.932898596820536e-05,
+      "loss": 0.6713860034942627,
+      "step": 70,
+      "token_acc": 0.8179179583556543
+    },
+    {
+      "epoch": 0.16541742052209873,
+      "grad_norm": 1.1796875,
+      "learning_rate": 1.9067005759998797e-05,
+      "loss": 0.6746496677398681,
+      "step": 80,
+      "token_acc": 0.8165457239789224
+    },
+    {
+      "epoch": 0.18609459808736106,
+      "grad_norm": 1.1953125,
+      "learning_rate": 1.876435729507959e-05,
+      "loss": 0.6496047496795654,
+      "step": 90,
+      "token_acc": 0.8231839497434082
+    },
+    {
+      "epoch": 0.20677177565262342,
+      "grad_norm": 1.0625,
+      "learning_rate": 1.8422398043134068e-05,
+      "loss": 0.6486006259918213,
+      "step": 100,
+      "token_acc": 0.822600955086533
+    },
+    {
+      "epoch": 0.22744895321788575,
+      "grad_norm": 1.1484375,
+      "learning_rate": 1.8042661794590023e-05,
+      "loss": 0.658829641342163,
+      "step": 110,
+      "token_acc": 0.8208090582207568
+    },
+    {
+      "epoch": 0.2481261307831481,
+      "grad_norm": 1.15625,
+      "learning_rate": 1.762685178110382e-05,
+      "loss": 0.6648715972900391,
+      "step": 120,
+      "token_acc": 0.8185997272110292
+    },
+    {
+      "epoch": 0.26880330834841043,
+      "grad_norm": 1.1640625,
+      "learning_rate": 1.7176833036052495e-05,
+      "loss": 0.665419340133667,
+      "step": 130,
+      "token_acc": 0.8191171753031502
+    },
+    {
+      "epoch": 0.28948048591367276,
+      "grad_norm": 1.234375,
+      "learning_rate": 1.6694624029296288e-05,
+      "loss": 0.661893653869629,
+      "step": 140,
+      "token_acc": 0.8190959299464042
+    },
+    {
+      "epoch": 0.31015766347893514,
+      "grad_norm": 1.1484375,
+      "learning_rate": 1.6182387613732295e-05,
+      "loss": 0.6551508903503418,
+      "step": 150,
+      "token_acc": 0.820616800353477
+    },
+    {
+      "epoch": 0.33083484104419747,
+      "grad_norm": 1.1796875,
+      "learning_rate": 1.564242132424657e-05,
+      "loss": 0.6404322624206543,
+      "step": 160,
+      "token_acc": 0.8242824359501302
+    },
+    {
+      "epoch": 0.3515120186094598,
+      "grad_norm": 1.1953125,
+      "learning_rate": 1.5077147072576932e-05,
+      "loss": 0.6450911521911621,
+      "step": 170,
+      "token_acc": 0.8225531631897226
+    },
+    {
+      "epoch": 0.3721891961747221,
+      "grad_norm": 1.109375,
+      "learning_rate": 1.4489100284308018e-05,
+      "loss": 0.6588228225708008,
+      "step": 180,
+      "token_acc": 0.8192937625626946
+    },
+    {
+      "epoch": 0.3928663737399845,
+      "grad_norm": 1.1953125,
+      "learning_rate": 1.3880918526722497e-05,
+      "loss": 0.6636178016662597,
+      "step": 190,
+      "token_acc": 0.8182313404409566
+    },
+    {
+      "epoch": 0.41354355130524684,
+      "grad_norm": 1.0703125,
+      "learning_rate": 1.3255329678515959e-05,
+      "loss": 0.6327512264251709,
+      "step": 200,
+      "token_acc": 0.8255940851586625
+    },
+    {
+      "epoch": 0.43422072887050917,
+      "grad_norm": 1.15625,
+      "learning_rate": 1.2615139694437835e-05,
+      "loss": 0.6687861442565918,
+      "step": 210,
+      "token_acc": 0.8168444004461097
+    },
+    {
+      "epoch": 0.4548979064357715,
+      "grad_norm": 1.171875,
+      "learning_rate": 1.1963220019737691e-05,
+      "loss": 0.6480214595794678,
+      "step": 220,
+      "token_acc": 0.8209949912628607
+    },
+    {
+      "epoch": 0.4755750840010339,
+      "grad_norm": 1.1328125,
+      "learning_rate": 1.1302494710866859e-05,
+      "loss": 0.6437870502471924,
+      "step": 230,
+      "token_acc": 0.8233039990630567
+    },
+    {
+      "epoch": 0.4962522615662962,
+      "grad_norm": 1.2109375,
+      "learning_rate": 1.0635927320202928e-05,
+      "loss": 0.6444666385650635,
+      "step": 240,
+      "token_acc": 0.8233064313791084
+    },
+    {
+      "epoch": 0.5169294391315585,
+      "grad_norm": NaN,
+      "learning_rate": 9.966507603623125e-06,
+      "loss": 0.6417760372161865,
+      "step": 250,
+      "token_acc": 0.8239854735118844
+    },
+    {
+      "epoch": 0.5376066166968209,
+      "grad_norm": 1.140625,
+      "learning_rate": 9.297238110547075e-06,
+      "loss": 0.6363988876342773,
+      "step": 260,
+      "token_acc": 0.8253960322908611
+    },
+    {
+      "epoch": 0.5582837942620832,
+      "grad_norm": 1.0390625,
+      "learning_rate": 8.631120716596642e-06,
+      "loss": 0.6291430950164795,
+      "step": 270,
+      "token_acc": 0.8267635377992593
+    },
+    {
+      "epoch": 0.5789609718273455,
+      "grad_norm": NaN,
+      "learning_rate": 7.971143159278025e-06,
+      "loss": 0.6549222946166993,
+      "step": 280,
+      "token_acc": 0.8202583345423531
+    },
+    {
+      "epoch": 0.599638149392608,
+      "grad_norm": 1.171875,
+      "learning_rate": 7.320265637077473e-06,
+      "loss": 0.6288355827331543,
+      "step": 290,
+      "token_acc": 0.8272131756443049
+    },
+    {
+      "epoch": 0.6203153269578703,
+      "grad_norm": 1.09375,
+      "learning_rate": 6.681407532077895e-06,
+      "loss": 0.6366195678710938,
+      "step": 300,
+      "token_acc": 0.8246204996799724
+    },
+    {
+      "epoch": 0.6409925045231326,
+      "grad_norm": 1.2265625,
+      "learning_rate": 6.057434315649304e-06,
+      "loss": 0.6529150485992432,
+      "step": 310,
+      "token_acc": 0.8214805383775918
+    },
+    {
+      "epoch": 0.6616696820883949,
+      "grad_norm": 1.125,
+      "learning_rate": 5.451144695945116e-06,
+      "loss": 0.6422211170196533,
+      "step": 320,
+      "token_acc": 0.8234491309794939
+    },
+    {
+      "epoch": 0.6823468596536573,
+      "grad_norm": 1.109375,
+      "learning_rate": 4.865258064851579e-06,
+      "loss": 0.6308234691619873,
+      "step": 330,
+      "token_acc": 0.8257183171934186
+    },
+    {
+      "epoch": 0.7030240372189196,
+      "grad_norm": 1.1953125,
+      "learning_rate": 4.302402300694636e-06,
+      "loss": 0.6352657794952392,
+      "step": 340,
+      "token_acc": 0.8245651251591005
+    },
+    {
+      "epoch": 0.7237012147841819,
+      "grad_norm": 1.1484375,
+      "learning_rate": 3.7651019814126656e-06,
+      "loss": 0.6413043022155762,
+      "step": 350,
+      "token_acc": 0.8234804751669123
+    },
+    {
+      "epoch": 0.7443783923494443,
+      "grad_norm": 1.15625,
+      "learning_rate": 3.2557670610626924e-06,
+      "loss": 0.6614915370941162,
+      "step": 360,
+      "token_acc": 0.8185413597228922
+    },
+    {
+      "epoch": 0.7650555699147067,
+      "grad_norm": 1.1875,
+      "learning_rate": 2.7766820604492474e-06,
+      "loss": 0.6162252426147461,
+      "step": 370,
+      "token_acc": 0.8291887765179509
+    },
+    {
+      "epoch": 0.785732747479969,
+      "grad_norm": 1.21875,
+      "learning_rate": 2.32999582035923e-06,
+      "loss": 0.6416737556457519,
+      "step": 380,
+      "token_acc": 0.8251579582098943
+    },
+    {
+      "epoch": 0.8064099250452313,
+      "grad_norm": 1.0625,
+      "learning_rate": 1.917711863362581e-06,
+      "loss": 0.6319101333618165,
+      "step": 390,
+      "token_acc": 0.8257550774349528
+    },
+    {
+      "epoch": 0.8270871026104937,
+      "grad_norm": 1.1796875,
+      "learning_rate": 1.5416794074090258e-06,
+      "loss": 0.6236296653747558,
+      "step": 400,
+      "token_acc": 0.8279304274286109
+    },
+    {
+      "epoch": 0.847764280175756,
+      "grad_norm": 1.109375,
+      "learning_rate": 1.2035850715275865e-06,
+      "loss": 0.6315475940704346,
+      "step": 410,
+      "token_acc": 0.8259817088278696
+    },
+    {
+      "epoch": 0.8684414577410183,
+      "grad_norm": 1.15625,
+      "learning_rate": 9.049453108312967e-07,
+      "loss": 0.64545578956604,
+      "step": 420,
+      "token_acc": 0.8220505314986443
+    },
+    {
+      "epoch": 0.8891186353062807,
+      "grad_norm": 1.2265625,
+      "learning_rate": 6.470996147584684e-07,
+      "loss": 0.6390243530273437,
+      "step": 430,
+      "token_acc": 0.8245470650344368
+    },
+    {
+      "epoch": 0.909795812871543,
+      "grad_norm": 1.078125,
+      "learning_rate": 4.3120449905836746e-07,
+      "loss": 0.6486499309539795,
+      "step": 440,
+      "token_acc": 0.8215761843927472
+    },
+    {
+      "epoch": 0.9304729904368054,
+      "grad_norm": 1.1171875,
+      "learning_rate": 2.5822831846912035e-07,
+      "loss": 0.6450016021728515,
+      "step": 450,
+      "token_acc": 0.8231799966516982
+    },
+    {
+      "epoch": 0.9511501680020678,
+      "grad_norm": 1.1015625,
+      "learning_rate": 1.2894692335451376e-07,
+      "loss": 0.6580901145935059,
+      "step": 460,
+      "token_acc": 0.8195172223593108
+    },
+    {
+      "epoch": 0.9718273455673301,
+      "grad_norm": 1.1171875,
+      "learning_rate": 4.394017978101905e-08,
+      "loss": 0.6284302234649658,
+      "step": 470,
+      "token_acc": 0.8261059191431867
+    },
+    {
+      "epoch": 0.9925045231325924,
+      "grad_norm": 1.09375,
+      "learning_rate": 3.5893686435028995e-09,
+      "loss": 0.637957763671875,
+      "step": 480,
+      "token_acc": 0.8241242127369184
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 484,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.135171100255191e+17,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

tools/build_manifest.py

@@ -19,6 +19,10 @@ SOURCE_PREFIXES = {
     "models/pretrained/gpt2_124m_code_5epoch": WORKSPACE / "llm.c/log_code_messages_5epoch/hf_export_gpt2_code_messages_epoch5_final",
     "models/pretrained/qwen25_coder_7b_cpt": WORKSPACE / "Pre_train/output/qwen25_coder_7b_full/v3-20260329-200947/checkpoint-18165",
     "models/sft/sft_f3_refined_instruct": WORKSPACE / "Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v2_1epoch_1gpu/v0-20260508-164342/checkpoint-6435",
+    "models/sft/sft_f3_clean_instruct": WORKSPACE / "Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_clean_1epoch_1gpu/v1-20260505-184601/checkpoint-14550",
+    "models/sft/sft_f3_hq_instruct": WORKSPACE / "Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v4_1epoch_1gpu/v0-20260519-205734/checkpoint-4212",
+    "models/sft/sft_f3f4_instruct": WORKSPACE / "Pre_train/output_user/qwen25_coder_0p5b_instruct_F3F4_clean_1epoch_1gpu/v0-20260506-103602/checkpoint-15033",
+    "models/sft/sft_f4_instruct": WORKSPACE / "Pre_train/output_user/qwen25_coder_0p5b_instruct_F4_clean_1epoch_1gpu/v3-20260505-141059/checkpoint-484",
     "code/f3_filtering": WORKSPACE / "Datasets/Instruction-turning/F3_v2_pipeline",
     "code/pretrain_data_prep": WORKSPACE / "Datasets",
     "code/llmc_training/llmc": WORKSPACE / "llm.c/llmc",
@@ -136,7 +140,7 @@ def main() -> None:
         files.append(record)
 
     manifest = {
-        "schema_version": "milestone-2-model-training-artifacts-v2",
+        "schema_version": "milestone-2-model-training-artifacts-v3",
         "generated_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"),
         "delivery_root": str(ROOT),
         "headline": {
@@ -146,7 +150,17 @@ def main() -> None:
                 "models/pretrained/qwen25_coder_7b_cpt",
                 "models/pretrained/stage1_cpt",
                 "models/sft/sft_f3_refined_instruct",
+                "models/sft/sft_f3_clean_instruct",
+                "models/sft/sft_f3_hq_instruct",
+                "models/sft/sft_f3f4_instruct",
+                "models/sft/sft_f4_instruct",
             ],
+            "selected_sft_variants": {
+                "S1": "models/sft/sft_f3_clean_instruct",
+                "S2": "models/sft/sft_f3_hq_instruct",
+                "S3": "models/sft/sft_f3f4_instruct",
+                "S4": "models/sft/sft_f4_instruct",
+            },
             "reported_pass_at_1_pct": 9.1,
             "reported_pass_at_5_pct": 13.0,
             "evaluation": "Runnable alignment benchmark; 200 tasks; 5 samples/task; 1000 attempts",
@@ -156,6 +170,8 @@ def main() -> None:
             "raw llm.c binary checkpoints and distributed optimizer states are not included",
             "the 20,021,101,140-byte raw dataset used by the three added CPT models is not duplicated",
             "the exact May 0.5B training environment snapshot was not captured",
+            "S2-S4 datasets and variant-specific launch scripts remain in local candidate staging",
+            "Magicoder experiments are excluded by delivery-owner decision",
             "CodeGPT-Multilingual external redistribution requires rights confirmation",
         ],
         "storage": {

tools/smoke_test.py

@@ -39,6 +39,9 @@ TARGETS = {
     "pretrained/stage1_cpt": ROOT / "models/pretrained/stage1_cpt",
     "sft/sft_f3_refined_instruct": ROOT / "models/sft/sft_f3_refined_instruct",
 }
+for checkpoint in sorted((ROOT / "models/sft").iterdir()):
+    if checkpoint.is_dir() and (checkpoint / "config.json").is_file():
+        TARGETS.setdefault(f"sft/{checkpoint.name}", checkpoint)
 
 
 def check(name: str, path: Path) -> dict: