exp-a retrain era_a1_refinedweb

exp-a-core-retrain/era_a1_refinedweb/run_config.json

@@ -0,0 +1,78 @@
+{
+  "size": "small",
+  "arm": "a1d0",
+  "data_glob": "/workspace/datasets/refinedweb_nanogpt/train_*.bin",
+  "neutral_eval_dir": "/workspace/datasets/wiki_eval_union",
+  "token_budget": 731381760,
+  "n_epochs": 1,
+  "out_dir": "/workspace/runs/era_a1_refinedweb",
+  "n_checkpoints": 25,
+  "first_ckpt_frac": 0.001,
+  "eval_seq_len": 1024,
+  "eval_windows_per_chunk": 32,
+  "val_batch_size": 2097152,
+  "model_max_seq_len": 0,
+  "seed": 1234,
+  "save_checkpoints": 1,
+  "keep_checkpoints": 3,
+  "lr": 0.023,
+  "adam_lr": 0.008,
+  "world_size": 8,
+  "total_steps": 2780,
+  "tokens_per_step": 263087,
+  "tokens_per_step_stages": [
+    [
+      920,
+      131072
+    ],
+    [
+      920,
+      262144
+    ],
+    [
+      940,
+      393216
+    ]
+  ],
+  "grad_accum": 1,
+  "n_params": 498773000,
+  "device": "cuda:0",
+  "gpu_name": "NVIDIA H100 80GB HBM3",
+  "torch_version": "2.10.0+cu128",
+  "algorithm": "new_modded_nanogpt",
+  "ckpt_steps": [
+    6,
+    8,
+    10,
+    14,
+    18,
+    24,
+    32,
+    42,
+    56,
+    75,
+    100,
+    133,
+    177,
+    236,
+    314,
+    419,
+    558,
+    745,
+    957,
+    1122,
+    1343,
+    1637,
+    1966,
+    2315,
+    2594,
+    2632,
+    2669,
+    2706,
+    2743,
+    2780
+  ],
+  "num_scheduled_iterations": 2760,
+  "num_extension_iterations": 20,
+  "total_tokens_trained": 731381760
+}