| { |
| "name": "final_latest", |
| "source_checkpoint": "/root/oeis_runs/oeis-440m-14b-full-20260525_025101/latest.pt", |
| "trained_tokens": 13999999995, |
| "trainer_state": { |
| "completed_steps": 581678, |
| "train_tokens_seen": 13999999995, |
| "last_loss": 0.5624260902404785 |
| }, |
| "checkpoint_args": { |
| "data": "/root/oeis-massive/packed_data/oeis_train_full_synth_plus_organic_13999999995.packed", |
| "model_backend": "custom", |
| "param_dtype": "fp32", |
| "weight_update_mode": "bf16_live_fp32_master", |
| "batch_size": 1, |
| "seq_len": 4096, |
| "steps": 5, |
| "warmup_steps": 2, |
| "target_tokens": 13999999995, |
| "max_steps": 0, |
| "batch_mode": "bucketed", |
| "pad_to_seq_len": false, |
| "index_dir": "", |
| "bucket_tokens_per_batch": 16384, |
| "bucket_token_budget_spec": "512:32768,*:24576", |
| "bucket_max_batch_size": 512, |
| "bucket_pad_multiple": 8, |
| "bucket_pad_to_upper": true, |
| "bucket_replacement": false, |
| "bucket_repeat_epochs": true, |
| "bucket_sampling": "token_mass", |
| "vocab_mode": "oeis", |
| "hidden_size": 1024, |
| "intermediate_size": 3072, |
| "num_hidden_layers": 28, |
| "num_attention_heads": 16, |
| "num_key_value_heads": 8, |
| "head_dim": 0, |
| "compile": true, |
| "compile_mode": "reduce-overhead", |
| "compile_dynamic": true, |
| "compile_skip_dynamic_cudagraphs": true, |
| "prewarm_bucket_shapes": true, |
| "prewarm_bucket_passes": 2, |
| "prewarm_restore_state": true, |
| "prewarm_verify_restore": true, |
| "prewarm_update_optimizer": true, |
| "prewarm_materialize_optimizer_state": true, |
| "gradient_checkpointing": false, |
| "native_gqa": true, |
| "amp": true, |
| "lr": 0.0003, |
| "optimizer_mode": "torch_muon_hybrid", |
| "adamw_lr": 0.0001, |
| "muon_lr": 0.01, |
| "body_lr_mult": 1.0, |
| "adamw_weight_decay": 0.01, |
| "muon_weight_decay": 0.0, |
| "muon_momentum": 0.95, |
| "muon_ns_steps": 5, |
| "muon_adjust_lr_fn": "", |
| "no_muon_nesterov": false, |
| "lr_schedule": "warmup_cosine_cooldown", |
| "lr_total_tokens": 13999999995, |
| "lr_warmup_tokens": 0, |
| "lr_warmup_fraction": 0.005, |
| "lr_decay_end_fraction": 0.95, |
| "lr_min_factor": 0.1, |
| "lr_final_factor": 0.0, |
| "checkpoint_dir": "/root/oeis_runs/oeis-440m-14b-full-20260525_025101", |
| "checkpoint_every_tokens": 500000000, |
| "checkpoint_every_steps": 0, |
| "keep_last_checkpoints": 4, |
| "resume": "/root/oeis_runs/oeis-440m-14b-full-20260525_025101/latest.pt", |
| "save_final": true, |
| "trim_final_batch": true, |
| "allow_token_overshoot": false, |
| "val_data": "/root/oeis_val_decontam.jsonl", |
| "val_format": "auto", |
| "val_index_dir": "", |
| "val_every_tokens": 500000000, |
| "val_every_steps": 0, |
| "val_batches": 256, |
| "val_batch_size": 32, |
| "val_max_examples": 0, |
| "val_max_context_tokens": 0, |
| "oeis_eval_data": "", |
| "oeis_eval_every_tokens": 0, |
| "oeis_eval_every_steps": 0, |
| "oeis_eval_batch_size": 64, |
| "oeis_eval_max_examples": 0, |
| "oeis_eval_max_new_tokens": 20, |
| "oeis_eval_max_context_tokens": 0, |
| "oeis_eval_collect_examples": 3, |
| "oeis_eval_generation_backend": "legacy", |
| "expected_loss_tokens": 0, |
| "safe_preflight": true, |
| "allow_synthetic_only": false, |
| "allow_replacement_sampling": false, |
| "preflight_only": false, |
| "wandb": true, |
| "wandb_project": "oeis-massive", |
| "wandb_entity": "n8programs", |
| "wandb_run_name": "oeis-440m-14b-full-20260525_025101", |
| "wandb_id": "oeis440m14b_20260525_025101", |
| "wandb_resume": "allow", |
| "wandb_mode": "online", |
| "wandb_tags": "full,440m,14b,resume_skipdyn", |
| "log_every_steps": 10, |
| "seed": 0, |
| "report_json": "/root/oeis_runs/oeis-440m-14b-full-20260525_025101/report_resume_skipdyn_20260526_114400.json" |
| }, |
| "transformers_compatible": true, |
| "rope_basis": "HF/Qwen split-half RoPE basis", |
| "source_rope_basis": "custom interleaved even/odd RoPE basis", |
| "conversion": "q_proj/k_proj rows and q_norm/k_norm weights permuted with run_oeis_nextterm_eval_torch._map_custom_state_to_transformers", |
| "oeis_vocab": { |
| "0-9": "digit tokens", |
| "10": "negative sign", |
| "11": "term separator", |
| "12": "BOS", |
| "13": "EOS", |
| "14": "PAD", |
| "15": "reserved" |
| }, |
| "generation_defaults": { |
| "max_context_tokens": 4096, |
| "recommended_max_new_tokens_for_oeis_eval_neo": 192, |
| "stop_token_ids": [ |
| 11, |
| 13, |
| 14 |
| ] |
| }, |
| "dtype": "bfloat16", |
| "elapsed_seconds": 7.9373568799346685, |
| "local_conversion": { |
| "source": "/Users/natebreslow/Documents/khashabiLab/bigOEIS/hf_downloads/NextTerm-440M-Checkpoints/checkpoints/final_latest", |
| "conversion": "float tensors downcast from fp32 safetensors to bf16 safetensors for local MLX inference throughput" |
| } |
| } |
|
|
