Upload 7 files

best/meta.json

@@ -0,0 +1,4 @@
+{
+  "step": 13500,
+  "best_val_loss": 3.1774327754974365
+}

best/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f97768e8a3703727804e4fcd96fefe60934b2b34372beab87c9dbcda37a160be
+size 133490789

best/model_config.json

@@ -0,0 +1,14 @@
+{
+  "vocab_size": 16000,
+  "n_layers": 8,
+  "n_heads": 8,
+  "n_kv_heads": 4,
+  "d_model": 512,
+  "d_ff": 1536,
+  "context_len": 1024,
+  "dropout": 0.0,
+  "bias": false,
+  "norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "tie_weights": true
+}

best/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ae45bd62440e2b9b749ede646a4e50c11061678507d6283ae2f7027d7081ebc
+size 266996363

best/train_config.json

@@ -0,0 +1,42 @@
+{
+  "corpus": "data/turkish_corpus.txt",
+  "val_corpus": "None",
+  "tokenizer_path": "tokenizer/",
+  "output_dir": "runs/tr45m_pretrain",
+  "cache_dir": ".cache",
+  "resume": "None",
+  "strategy": "ddp",
+  "ds_stage": "2",
+  "cpu_offload": "False",
+  "preset": "small",
+  "n_layers": "12",
+  "n_heads": "12",
+  "n_kv_heads": "4",
+  "d_model": "768",
+  "context_len": "None",
+  "dropout": "None",
+  "no_tie_weights": "False",
+  "compile": "False",
+  "batch_size": "32",
+  "grad_accum": "2",
+  "max_steps": "13700",
+  "warmup_steps": "400",
+  "lr": "0.0003",
+  "min_lr": "3e-05",
+  "weight_decay": "0.1",
+  "beta1": "0.9",
+  "beta2": "0.95",
+  "grad_clip": "1.0",
+  "mixed_precision": "bf16",
+  "streaming": "False",
+  "num_workers": "4",
+  "seed": "42",
+  "log_every": "50",
+  "eval_every": "500",
+  "eval_batches": "100",
+  "save_every": "1000",
+  "wandb": "False",
+  "wandb_project": "my-llm",
+  "sample_prompt": "T\u00fcrkiye'nin ba\u015fkenti",
+  "sample_tokens": "100"
+}

tokenizer/tokenizer_meta.json

@@ -0,0 +1,18 @@
+{
+  "vocab_size": 16000,
+  "special_tokens": {
+    "<|bos|>": 0,
+    "<|eos|>": 1,
+    "<|pad|>": 2,
+    "<|sep|>": 3,
+    "<|unk|>": 4,
+    "<|sys|>": 5,
+    "<|usr|>": 6,
+    "<|ast|>": 7
+  },
+  "bos_id": 0,
+  "eos_id": 1,
+  "pad_id": 2,
+  "sep_id": 3,
+  "unk_id": 4
+}