Upload folder using huggingface_hub

README.md

@@ -0,0 +1,45 @@
+---
+license: apache-2.0
+base_model: mistralai/Mistral-Small-3.1-24B-Instruct-2503
+tags:
+- lora
+- mlx
+- mixture-of-experts
+- code-generation
+- python
+- typescript
+---
+
+# FrankenMoE: Python + TypeScript Expert LoRA Adapters
+
+Dual LoRA adapters ที่ fine-tune บน 4-bit quantized MLX version ของ
+`mistralai/Mistral-Small-3.1-24B-Instruct-2503` แยกตามภาษาโปรแกรมมิ่ง
+พร้อม Dispatcher script สำหรับ hot-swap ระหว่าง Expert แบบไม่ต้องโหลด
+Base Model ใหม่
+
+## Experts
+
+| Expert | Language | Rank | Target Modules | Checkpoint |
+|---|---|---|---|---|
+| expert-1-python-v2 | Python | 16 | q,k,v,o_proj | iter 500 |
+| expert-2-typescript | TypeScript | 8 | default (incl. MoE switch layers) | iter 100 |
+
+## Usage
+
+ต้องมี Base Model 4-bit อยู่ในเครื่อง (หรือดาวน์โหลดจาก [link ต้นทาง])
+
+\`\`\`bash
+python frankenmoe_dispatcher.py
+\`\`\`
+
+Dispatcher จะตรวจจับภาษาจาก Prompt และสลับ LoRA Adapter อัตโนมัติ
+โดยจัดการทั้ง \`LoRALinear\` (attention) และ \`LoRASwitchLinear\`
+(MoE expert MLP) อย่างถูกต้อง
+
+## Training Details
+
+- **Expert 1 (Python)**: แก้ไขปัญหา rank/capacity ไม่พอ (v1 broken ที่ rank=8)
+  โดยอัปเกรดเป็น rank=16, scale=8.0, ครอบคลุม 4 attention projections
+- **Expert 2 (TypeScript)**: แก้ไข NaN loss (SAFE_PAD_TOKEN_ID=198) และ
+  Catastrophic Interference ระหว่าง function/type guard bias ผ่าน
+  dataset rebalancing และลด learning rate เป็น 1.5e-5

adapters/expert-1-python-v2/adapter_config.json

@@ -0,0 +1,46 @@
+{
+    "adapter_path": "./adapters/expert-1-python-v2",
+    "batch_size": 1,
+    "config": "lora_config_expert1_v2.yaml",
+    "data": "./datasets/expert-1-python",
+    "fine_tune_type": "lora",
+    "grad_accumulation_steps": 1,
+    "grad_checkpoint": true,
+    "iters": 500,
+    "learning_rate": 1e-05,
+    "lora_parameters": {
+        "rank": 16,
+        "dropout": 0.0,
+        "scale": 8.0,
+        "keys": [
+            "self_attn.q_proj",
+            "self_attn.k_proj",
+            "self_attn.v_proj",
+            "self_attn.o_proj"
+        ]
+    },
+    "lr_schedule": null,
+    "mask_prompt": false,
+    "max_seq_length": 2048,
+    "model": "./output-moe-mlx-4bit",
+    "num_layers": 16,
+    "optimizer": "adam",
+    "optimizer_config": {
+        "adam": {},
+        "adamw": {},
+        "muon": {},
+        "sgd": {},
+        "adafactor": {}
+    },
+    "project_name": null,
+    "report_to": null,
+    "resume_adapter_file": null,
+    "save_every": 50,
+    "seed": 0,
+    "steps_per_eval": 200,
+    "steps_per_report": 10,
+    "test": false,
+    "test_batches": 500,
+    "train": true,
+    "val_batches": 25
+}

adapters/expert-1-python-v2/adapters.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf27527b12cf66f6c375b67b015b70bf0fead0a89b08b09dbc519f92956f17f8
+size 33568580

adapters/expert-2-typescript/adapter_config.json

@@ -0,0 +1,40 @@
+{
+    "adapter_path": "adapters/expert-2-typescript",
+    "batch_size": 1,
+    "config": null,
+    "data": "datasets/expert-2-typescript",
+    "fine_tune_type": "lora",
+    "grad_accumulation_steps": 1,
+    "grad_checkpoint": false,
+    "iters": 150,
+    "learning_rate": 1e-05,
+    "lora_parameters": {
+        "rank": 8,
+        "dropout": 0.0,
+        "scale": 20.0
+    },
+    "lr_schedule": null,
+    "mask_prompt": false,
+    "max_seq_length": 2048,
+    "model": "output-moe-mlx-4bit",
+    "num_layers": 16,
+    "optimizer": "adam",
+    "optimizer_config": {
+        "adam": {},
+        "adamw": {},
+        "muon": {},
+        "sgd": {},
+        "adafactor": {}
+    },
+    "project_name": null,
+    "report_to": null,
+    "resume_adapter_file": "adapters/expert-2-typescript/0000400_adapters.safetensors",
+    "save_every": 25,
+    "seed": 0,
+    "steps_per_eval": 25,
+    "steps_per_report": 10,
+    "test": false,
+    "test_batches": 500,
+    "train": true,
+    "val_batches": 25
+}

adapters/expert-2-typescript/adapters.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0aba896a07bdc398e29920ab40e69f35812e61d396c79a26952c604e86c611c
+size 126137095

frankenmoe_dispatcher.py

@@ -0,0 +1,161 @@
+"""
+FrankenMoE Dispatcher — Rebuild-on-Swap Architecture (v4: Handle MoE Switch Layers)
+แก้ปัญหา 'Can't convert layer of type LoRASwitchLinear to LoRA'
+โดยตรวจจับทั้ง LoRALinear (Attention) และ LoRASwitchLinear (MoE Expert MLP)
+"""
+import json
+from mlx.utils import tree_unflatten
+from mlx_lm import load, generate
+from mlx_lm.tuner.utils import linear_to_lora_layers
+
+# 🔑 Import ทั้ง 2 Class ที่ mlx_lm ใช้ห่อ LoRA
+# LoRALinear      -> ใช้กับ Attention Layer ปกติ (q,k,v,o_proj)
+# LoRASwitchLinear -> ใช้กับ MoE Switch/Expert MLP Layer โดยเฉพาะ
+LORA_WRAPPER_TYPES = []
+try:
+    from mlx_lm.tuner.lora import LoRALinear
+    LORA_WRAPPER_TYPES.append(LoRALinear)
+except ImportError:
+    pass
+try:
+    from mlx_lm.tuner.lora import LoRASwitchLinear
+    LORA_WRAPPER_TYPES.append(LoRASwitchLinear)
+except ImportError:
+    pass
+
+LORA_WRAPPER_TYPES = tuple(LORA_WRAPPER_TYPES)
+print(f"🔎 ตรวจพบ LoRA Wrapper Types ที่ต้องจัดการ: {[t.__name__ for t in LORA_WRAPPER_TYPES]}")
+
+
+class FrankenMoEDispatcher:
+    def __init__(self, base_model_path="output-moe-mlx-4bit", max_num_layers=16):
+        print(f"🔄 กำลังโหลด Base Model: {base_model_path} ...")
+        self.model, self.tokenizer = load(base_model_path)
+        self.max_num_layers = max_num_layers
+        self.current_expert = None
+
+        self.experts = {
+            "python": {
+                "config_path": "adapters/expert-1-python-v2/adapter_config.json",
+                "weights_path": "adapters/expert-1-python-v2/0000500_adapters.safetensors",
+            },
+            "typescript": {
+                "config_path": "adapters/expert-2-typescript/adapter_config.json",
+                "weights_path": "adapters/expert-2-typescript/0000100_adapters.safetensors",
+            },
+        }
+
+        # Snapshot ทุก Module ใน Layer เป้าหมาย (Attention + MoE Switch + อะไรก็ตาม)
+        self._original_layers = {}
+        total_layers = len(self.model.layers)
+        self._layer_range = range(total_layers - max_num_layers, total_layers)
+        for real_idx in self._layer_range:
+            l = self.model.layers[real_idx]
+            for k, m in l.named_modules():
+                self._original_layers[(real_idx, k)] = m
+
+        print(f"✅ Base Model พร้อมใช้งาน (Snapshot {len(self._original_layers)} modules)\n")
+
+    def _restore_original_layers(self):
+        """สแกนหา Module ที่เป็น LoRALinear หรือ LoRASwitchLinear แล้ว Restore กลับเป็นของเดิม"""
+        restored_count = 0
+        restored_types = {}
+        for real_idx in self._layer_range:
+            l = self.model.layers[real_idx]
+            restore_pairs = []
+            for k, m in l.named_modules():
+                if isinstance(m, LORA_WRAPPER_TYPES):
+                    orig = self._original_layers.get((real_idx, k))
+                    if orig is not None:
+                        restore_pairs.append((k, orig))
+                        restored_count += 1
+                        type_name = type(m).__name__
+                        restored_types[type_name] = restored_types.get(type_name, 0) + 1
+            if restore_pairs:
+                l.update_modules(tree_unflatten(restore_pairs))
+        return restored_count, restored_types
+
+    def load_expert(self, expert_name: str):
+        if expert_name == self.current_expert:
+            return
+
+        if expert_name not in self.experts:
+            raise ValueError(f"ไม่รู้จัก Expert: {expert_name}")
+
+        info = self.experts[expert_name]
+        with open(info["config_path"]) as f:
+            adapter_config = json.load(f)
+
+        num_layers = adapter_config.get("num_layers", 16)
+        lora_parameters = adapter_config.get("lora_parameters", {})
+
+        print(f"🔧 กำลังสลับไปใช้ Expert: '{expert_name}' "
+              f"(rank={lora_parameters.get('rank')}, "
+              f"scale={lora_parameters.get('scale')}, "
+              f"keys={lora_parameters.get('keys', 'default')})")
+
+        n, types_breakdown = self._restore_original_layers()
+        print(f"   ↳ Restored {n} module(s) กลับเป็นของเดิม | breakdown: {types_breakdown}")
+
+        linear_to_lora_layers(self.model, num_layers, lora_parameters, use_dora=False)
+        self.model.load_weights(info["weights_path"], strict=False)
+        self.model.eval()
+
+        self.current_expert = expert_name
+        print(f"✅ พร้อมใช้งาน Expert: '{expert_name}'\n")
+
+    def route(self, prompt: str) -> str:
+        ts_signals = [
+            "typescript", "interface", "type guard", "generic", ": string",
+            ": number", "utility type", "conditional type", "mapped type",
+        ]
+        py_signals = [
+            "python", "def ", "decorator", "list comprehension",
+            "generator", "context manager",
+        ]
+        prompt_lower = prompt.lower()
+        ts_score = sum(1 for kw in ts_signals if kw in prompt_lower)
+        py_score = sum(1 for kw in py_signals if kw in prompt_lower)
+        return "typescript" if ts_score > py_score else "python"
+
+    def generate_response(self, prompt: str, max_tokens: int = 300):
+        expert_name = self.route(prompt)
+        self.load_expert(expert_name)
+
+        messages = [{"role": "user", "content": prompt}]
+        formatted_prompt = self.tokenizer.apply_chat_template(
+            messages, add_generation_prompt=True, tokenize=False
+        )
+        response = generate(
+            self.model, self.tokenizer,
+            prompt=formatted_prompt, max_tokens=max_tokens, verbose=False
+        )
+        return expert_name, response
+
+
+if __name__ == "__main__":
+    dispatcher = FrankenMoEDispatcher()
+
+    test_cases = [
+        "Write a Python function to check if a number is prime.",
+        "Write a TypeScript type guard function called isArray.",
+        "Write a Python decorator that logs execution time.",
+        "Create a TypeScript utility type Readonly that makes properties readonly recursively.",
+        "Write a Python generator function that yields Fibonacci numbers.",
+        "Write a TypeScript function isString that acts as a type guard.",
+        "Write a Python function using list comprehension to flatten a nested list.",
+        "Write a TypeScript debounce function using closures.",
+    ]
+
+    for i, prompt in enumerate(test_cases, 1):
+        print("=" * 70)
+        print(f"โจทย์ที่ {i}: {prompt}")
+        print("=" * 70)
+        expert_used, response = dispatcher.generate_response(prompt)
+        print(f"[Router เลือก Expert: {expert_used}]")
+        print(response)
+        print()
+
+    print("=" * 70)
+    print("🎉 สลับ Expert สำเร็จทั้งหมด 8 รอบ โดยไม่มี Error")
+    print("=" * 70)