best/train_config.json

{
  "corpus": "data/turkish_corpus.txt",
  "val_corpus": "None",
  "tokenizer_path": "tokenizer/",
  "output_dir": "runs/tr45m_pretrain",
  "cache_dir": ".cache",
  "resume": "None",
  "strategy": "ddp",
  "ds_stage": "2",
  "cpu_offload": "False",
  "preset": "small",
  "n_layers": "12",
  "n_heads": "12",
  "n_kv_heads": "4",
  "d_model": "768",
  "context_len": "None",
  "dropout": "None",
  "no_tie_weights": "False",
  "compile": "False",
  "batch_size": "32",
  "grad_accum": "2",
  "max_steps": "13700",
  "warmup_steps": "400",
  "lr": "0.0003",
  "min_lr": "3e-05",
  "weight_decay": "0.1",
  "beta1": "0.9",
  "beta2": "0.95",
  "grad_clip": "1.0",
  "mixed_precision": "bf16",
  "streaming": "False",
  "num_workers": "4",
  "seed": "42",
  "log_every": "50",
  "eval_every": "500",
  "eval_batches": "100",
  "save_every": "1000",
  "wandb": "False",
  "wandb_project": "my-llm",
  "sample_prompt": "T\u00fcrkiye'nin ba\u015fkenti",
  "sample_tokens": "100"
}