{
  "_comment": "NeuralAI Training Configuration v2.0 - From-Scratch GPT + QLoRA Fine-Tuning Pipeline",
  "version": "2.0.0",
  "project": "NeuralAI-from-scratch",

  "model": {
    "base_model": "HuggingFaceTB/SmolLM2-360M-Instruct",
    "model_type": "causal_lm",
    "architecture": "GPT-style Transformer (from-scratch PyTorch)",
    "quantization": {
      "load_in_4bit": true,
      "bnb_4bit_quant_type": "nf4",
      "bnb_4bit_compute_dtype": "bfloat16",
      "bnb_4bit_use_double_quant": true
    },
    "torch_dtype": "float16",
    "device_map": "auto",
    "trust_remote_code": false
  },

  "lora": {
    "r": 16,
    "lora_alpha": 32,
    "lora_dropout": 0.05,
    "bias": "none",
    "task_type": "CAUSAL_LM",
    "target_modules": [
      "q_proj",
      "k_proj",
      "v_proj",
      "o_proj",
      "gate_proj",
      "up_proj",
      "down_proj"
    ],
    "modules_to_save": null,
    "use_rslora": false
  },

  "training": {
    "output_dir": "./checkpoints",
    "num_train_epochs": 3,
    "per_device_train_batch_size": 4,
    "per_device_eval_batch_size": 4,
    "gradient_accumulation_steps": 4,
    "gradient_checkpointing": true,
    "learning_rate": 2e-4,
    "weight_decay": 0.01,
    "max_grad_norm": 1.0,
    "warmup_ratio": 0.1,
    "lr_scheduler_type": "cosine",
    "fp16": false,
    "bf16": true,
    "optim": "paged_adamw_32bit",
    "logging_steps": 10,
    "eval_steps": 50,
    "save_steps": 100,
    "save_total_limit": 3,
    "load_best_model_at_end": true,
    "evaluation_strategy": "steps",
    "report_to": "none",
    "seed": 42,
    "dataloader_num_workers": 2,
    "remove_unused_columns": false,
    "group_by_length": true
  },

  "data": {
    "train_file": "./data/train.jsonl",
    "val_file": "./data/val.jsonl",
    "text_column": "text",
    "chat_template": "chatml",
    "max_seq_length": 2048,
    "packing": false,
    "dataset_num_proc": 4
  },

  "model_configs": {
    "_comment": "From-scratch GPT tier configs (no HuggingFace, pure PyTorch)",
    "nano": {
      "n_layer": 2,
      "n_head": 2,
      "n_embd": 64,
      "context_length": 128,
      "params": "~2M",
      "vram": "<512MB",
      "use_case": "Testing / CI"
    },
    "4gb": {
      "n_layer": 4,
      "n_head": 4,
      "n_embd": 256,
      "context_length": 256,
      "params": "~8M",
      "vram": "<2GB",
      "use_case": "CPU / 4GB RAM laptop"
    },
    "small": {
      "n_layer": 6,
      "n_head": 6,
      "n_embd": 384,
      "context_length": 512,
      "params": "~30M",
      "vram": "~4GB",
      "use_case": "Colab T4 / single GPU"
    },
    "medium": {
      "n_layer": 12,
      "n_head": 12,
      "n_embd": 768,
      "context_length": 1024,
      "params": "~124M",
      "vram": "~8GB",
      "use_case": "GPT-2 equivalent"
    },
    "large": {
      "n_layer": 24,
      "n_head": 16,
      "n_embd": 1024,
      "context_length": 2048,
      "params": "~345M",
      "vram": "~24GB",
      "use_case": "GPT-2 Medium equivalent"
    }
  },

  "generation": {
    "max_new_tokens": 512,
    "temperature": 0.7,
    "top_p": 0.9,
    "top_k": 50,
    "repetition_penalty": 1.2,
    "do_sample": true,
    "use_kv_cache": true,
    "streaming": true
  },

  "features": {
    "flash_attention": true,
    "rope_embeddings": true,
    "kv_cache": true,
    "gradient_checkpointing": true,
    "swiglu_ffn": true,
    "weight_tying": true,
    "int8_inference": true,
    "streaming_generation": true,
    "web_ui": true,
    "colab_notebook": true
  },

  "paths": {
    "checkpoints_dir": "./checkpoints",
    "data_dir": "./data",
    "logs_dir": "./logs",
    "from_scratch_dir": "./from-scratch",
    "colab_notebook": "./NeuralAI_Colab_Training.ipynb"
  },

  "deployment": {
    "push_to_hub": false,
    "hub_model_id": "Subject-Emu-5259/NeuralAI",
    "hub_private": true,
    "merge_adapter_on_save": false,
    "export_onnx": false
  }
}