| { |
| "corpus": "data/turkish_corpus.txt", |
| "val_corpus": "None", |
| "tokenizer_path": "tokenizer/", |
| "output_dir": "runs/tr45m_pretrain", |
| "cache_dir": ".cache", |
| "resume": "None", |
| "strategy": "ddp", |
| "ds_stage": "2", |
| "cpu_offload": "False", |
| "preset": "small", |
| "n_layers": "12", |
| "n_heads": "12", |
| "n_kv_heads": "4", |
| "d_model": "768", |
| "context_len": "None", |
| "dropout": "None", |
| "no_tie_weights": "False", |
| "compile": "False", |
| "batch_size": "32", |
| "grad_accum": "2", |
| "max_steps": "13700", |
| "warmup_steps": "400", |
| "lr": "0.0003", |
| "min_lr": "3e-05", |
| "weight_decay": "0.1", |
| "beta1": "0.9", |
| "beta2": "0.95", |
| "grad_clip": "1.0", |
| "mixed_precision": "bf16", |
| "streaming": "False", |
| "num_workers": "4", |
| "seed": "42", |
| "log_every": "50", |
| "eval_every": "500", |
| "eval_batches": "100", |
| "save_every": "1000", |
| "wandb": "False", |
| "wandb_project": "my-llm", |
| "sample_prompt": "T\u00fcrkiye'nin ba\u015fkenti", |
| "sample_tokens": "100" |
| } |
