Add files using upload-large-folder tool

.gitattributes

@@ -36,3 +36,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 data/f3/intermediate/F3_stage2.jsonl filter=lfs diff=lfs merge=lfs -text
 data/f3/intermediate/F3_stage1.jsonl filter=lfs diff=lfs merge=lfs -text
 data/f3/intermediate/F3_stage3.jsonl filter=lfs diff=lfs merge=lfs -text
+models/sft/sft_f3_refined_instruct/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+models/pretrained/stage1_cpt/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+data/f3/output/scicode_sft_swift_F3_clean_v2.jsonl filter=lfs diff=lfs merge=lfs -text
+data/pretrain/code_messages_pre_train_filtered_stage1.jsonl filter=lfs diff=lfs merge=lfs -text

code/f3_filtering/reports/F3_stage4_report.json

@@ -0,0 +1,28 @@
+{
+  "src": "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_stage3.jsonl",
+  "dst": "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_stage4.jsonl",
+  "model": "/mnt/moon-data/jfjiang/swf/skillsecure/data/models/Qwen3-Coder-30B-A3B-Instruct",
+  "threshold": 3,
+  "n_total": 362367,
+  "n_kept": 207009,
+  "n_dropped": 155358,
+  "drop_rate": 0.4287,
+  "n_skipped_too_long_kept_default": 115,
+  "score_distribution": {
+    "0": 115,
+    "1": 10545,
+    "2": 144813,
+    "3": 159159,
+    "4": 47474,
+    "5": 261
+  },
+  "score_distribution_pct": {
+    "0": 0.03,
+    "1": 2.91,
+    "2": 39.96,
+    "3": 43.92,
+    "4": 13.1,
+    "5": 0.07
+  },
+  "elapsed_sec": 5039.8
+}

code/f3_filtering/reports/F3_v4_report.json

@@ -0,0 +1,24 @@
+{
+  "input": "/raid/data/weifeng/Datasets/Instruction-turning/scicode_sft_swift_F3_clean_v3.jsonl",
+  "output": "/raid/data/weifeng/Datasets/Instruction-turning/scicode_sft_swift_F3_clean_v4.jsonl",
+  "input_size": 154103,
+  "final_size": 134775,
+  "kept_rate": 0.8746,
+  "rules": [
+    "S1: MinHash near-dup, Jaccard >= 0.85, bottom-128 shingle-8",
+    "S2: template ctor / setup / main with trivial body (content-based)",
+    "S3: trivial body (< 3 non-trivial statements)",
+    "S4: comment+docstring ratio > 0.7",
+    "S6: import-only / docstring-only assistant",
+    "S8: placeholder residue (TODO, FIXME, XXX, HACK, pdb.set_trace, ...)"
+  ],
+  "applied_drops": {
+    "S1_near_dup": 6390,
+    "S3_trivial": 6376,
+    "S8_placeholder": 3958,
+    "S4_comment_ratio": 2286,
+    "S2_template": 318
+  },
+  "total_dropped": 19328,
+  "elapsed_sec": 101.0
+}

code/f3_filtering/reports/PIPELINE_REPORT.json

@@ -0,0 +1,76 @@
+{
+  "final_output": "/raid/data/weifeng/Datasets/Instruction-turning/scicode_sft_swift_F3_clean_v2.jsonl",
+  "input": "/raid/data/weifeng/Datasets/Instruction-turning/scicode_sft_swift_F3_clean.jsonl",
+  "input_size": 473465,
+  "final_size": 207009,
+  "kept_rate": 0.4373,
+  "dropped_total": 266456,
+  "drop_rate_total": 0.5627,
+  "stages": [
+    {
+      "stage": 1,
+      "name": "structural filters",
+      "rules": [
+        "1a: Python AST must parse (other languages skip this)",
+        "1b: drop if any line >=5 chars repeats >=5 times",
+        "1c: drop if <3 lines or >500 lines",
+        "1d: drop hardcoded local paths",
+        "1e: drop pure placeholder bodies (Python only)",
+        "1f: drop debug residue (pdb, breakpoint, >=3 TODO/FIXME)"
+      ],
+      "n_in": 473465,
+      "n_out": 437671,
+      "n_dropped": 35794,
+      "kept_languages": {
+        "python": 400263,
+        "cpp": 16733,
+        "c": 10995,
+        "java": 6879,
+        "rust": 2577,
+        "go": 224
+      }
+    },
+    {
+      "stage": 2,
+      "name": "per-project cap=300 with D1xD2 stratified sampling",
+      "rules": [
+        "D1 buckets (code lines): S<15, M1=15-39, M2=40-99, L>=100",
+        "D2 buckets (line repeat): norep<5, rep>=5 (after stage1 mostly empty)",
+        "Per project: target 75 per D1; rep cap=5 per D1; reallocate shortfall",
+        "Within bucket: random (seed=0)",
+        "Only projects with >300 samples are capped (217 of 14408)"
+      ],
+      "n_in": 437671,
+      "n_out": 366672,
+      "n_dropped": 70999
+    },
+    {
+      "stage": 3,
+      "name": "per-project assistant deduplication, K=3",
+      "n_in": 366672,
+      "n_out": 362367,
+      "n_dropped": 4305
+    },
+    {
+      "stage": 4,
+      "name": "Qwen3-Coder-30B-A3B-Instruct quality scoring (1-5)",
+      "rules": [
+        "threshold: keep score >= 3",
+        "115 samples too long for 8192 max_model_len, kept by default"
+      ],
+      "model": "/mnt/moon-data/jfjiang/swf/skillsecure/data/models/Qwen3-Coder-30B-A3B-Instruct",
+      "n_in": 362367,
+      "n_out": 207009,
+      "n_dropped": 155358,
+      "score_distribution": {
+        "1": 10545,
+        "2": 144813,
+        "3": 159159,
+        "4": 47474,
+        "5": 261,
+        "too_long_kept": 115
+      },
+      "elapsed_min": 84
+    }
+  ]
+}

data/f3/output/scicode_sft_swift_F3_clean_v2.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa3521aaac5ed5cb1575a2f92c0386cec598002175aa47dbbaaaf71b6aad5ac7
+size 1125890366

data/pretrain/code_messages_pre_train_filtered_stage1.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e582cefbfb377aee6f3cc0c23f8159d5a77b14de27cf9fb2c4722078b905459a
+size 1657338573

models/pretrained/stage1_cpt/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c119b284587312405231891869c326215379ec62890899a0b6af324204f8d290
+size 988097824

models/pretrained/stage1_cpt/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892

models/sft/sft_f3_refined_instruct/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05bca1c3ba20caad5a14ef0f450b6788aff7c9594bf90fe6deb5ecd2c5561847
+size 988097824

models/sft/sft_f3_refined_instruct/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892