diff --git a/.gitattributes b/.gitattributes index a7ecfd7073d4535cd5c23d12a6aece40cd962572..f084c0e12e1f802788bc13f276b2ba02fcbd027d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -37,3 +37,4 @@ wandb/run-20260226_135602-696nxyfr/run-696nxyfr.wandb filter=lfs diff=lfs merge= wandb/run-20260226_153026-trcpjlfd/run-trcpjlfd.wandb filter=lfs diff=lfs merge=lfs -text wandb/run-20260319_063518-29lbcxak/run-29lbcxak.wandb filter=lfs diff=lfs merge=lfs -text wandb/run-20260319_091054-lisp43b6/run-lisp43b6.wandb filter=lfs diff=lfs merge=lfs -text +wandb/run-20260325_092121-4guua5vm/run-4guua5vm.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/checkpoints/metadata_000000032768.json b/checkpoints/metadata_000000032768.json index d0e99ce0c6f570a854b2b114aea6a89a7532fce7..c8c3b7ec6823f9b7f3c05979af50dccdb1da6f51 100644 --- a/checkpoints/metadata_000000032768.json +++ b/checkpoints/metadata_000000032768.json @@ -1 +1 @@ -{"step": 1, "tokens_seen": 32768, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.462437629699707} \ No newline at end of file +{"step": 1, "tokens_seen": 32768, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.490738868713379} \ No newline at end of file diff --git a/checkpoints/metadata_000000327680.json b/checkpoints/metadata_000000327680.json index 20e027cc76f07f371a4058de1c753a16439a42e5..9b283f77c3845437fafd5da4a11446e75bdc8078 100644 --- a/checkpoints/metadata_000000327680.json +++ b/checkpoints/metadata_000000327680.json @@ -1 +1 @@ -{"step": 10, "tokens_seen": 327680, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.44581323632758} \ No newline at end of file +{"step": 10, "tokens_seen": 327680, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.488559456099267} \ No newline at end of file diff --git a/checkpoints/metadata_000000360448.json b/checkpoints/metadata_000000360448.json index 44a19446c1116d33f61ddf595abc0fbb446109c1..08dffc69efb6053e170178bcdea405b3519d7c7a 100644 --- a/checkpoints/metadata_000000360448.json +++ b/checkpoints/metadata_000000360448.json @@ -1 +1 @@ -{"step": 11, "tokens_seen": 360448, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.441161565298534} \ No newline at end of file +{"step": 11, "tokens_seen": 360448, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.487681315456733} \ No newline at end of file diff --git a/checkpoints/metadata_000000425984.json b/checkpoints/metadata_000000425984.json index 28e3c55176e21b7151e870bf8c0d8b69dd85e8fa..1aafd23814fbbb7c2a07fe2212759a30e9dc77a9 100644 --- a/checkpoints/metadata_000000425984.json +++ b/checkpoints/metadata_000000425984.json @@ -1 +1 @@ -{"step": 13, "tokens_seen": 425984, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.429075783774888} \ No newline at end of file +{"step": 13, "tokens_seen": 425984, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.485140885608613} \ No newline at end of file diff --git a/checkpoints/metadata_000000458752.json b/checkpoints/metadata_000000458752.json index b21c924d5f90feb3306d0f8f66bdd96f8c932f05..a8c03f8afcc3f57fd403bca4a21196c1fdae000c 100644 --- a/checkpoints/metadata_000000458752.json +++ b/checkpoints/metadata_000000458752.json @@ -1 +1 @@ -{"step": 28, "tokens_seen": 458752, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 10.378429363888989} \ No newline at end of file +{"step": 14, "tokens_seen": 458752, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.48362769925619} \ No newline at end of file diff --git a/checkpoints/metadata_000000491520.json b/checkpoints/metadata_000000491520.json index 47e03e9231ba61813ab4e81cbf0d11385ccb01c8..9f928751687bfe47b2a1002910147eeb393cdca1 100644 --- a/checkpoints/metadata_000000491520.json +++ b/checkpoints/metadata_000000491520.json @@ -1 +1 @@ -{"step": 30, "tokens_seen": 491520, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 10.362496783724538} \ No newline at end of file +{"step": 15, "tokens_seen": 491520, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.481805614844804} \ No newline at end of file diff --git a/checkpoints/metadata_000000557056.json b/checkpoints/metadata_000000557056.json index 9b147ed4b6bdf4854aed0783617384fa672156d1..010a64c4cb88014c837e9b9c5905bbc6d71b492e 100644 --- a/checkpoints/metadata_000000557056.json +++ b/checkpoints/metadata_000000557056.json @@ -1 +1 @@ -{"step": 34, "tokens_seen": 557056, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 10.329005356493877} \ No newline at end of file +{"step": 17, "tokens_seen": 557056, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.477386701187243} \ No newline at end of file diff --git a/checkpoints/metadata_000000622592.json b/checkpoints/metadata_000000622592.json index d279f8237609739ffeae3ed088ed1a18687cef4e..6f692adfd9351729bad558a3d0ec82b66028ad4a 100644 --- a/checkpoints/metadata_000000622592.json +++ b/checkpoints/metadata_000000622592.json @@ -1 +1 @@ -{"step": 19, "tokens_seen": 622592, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.387243368149452} \ No newline at end of file +{"step": 19, "tokens_seen": 622592, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.472137723118012} \ No newline at end of file diff --git a/checkpoints/metadata_000000688128.json b/checkpoints/metadata_000000688128.json index f3457162b0ed773047d16fa01b73f38e88a98baa..061e0c79842ece92f997efa99b1f03a7fba6e1b5 100644 --- a/checkpoints/metadata_000000688128.json +++ b/checkpoints/metadata_000000688128.json @@ -1 +1 @@ -{"step": 21, "tokens_seen": 688128, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.37150742909514} \ No newline at end of file +{"step": 21, "tokens_seen": 688128, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.465674521810133} \ No newline at end of file diff --git a/checkpoints/metadata_000000753664.json b/checkpoints/metadata_000000753664.json index 6c07b74fc081100e070b3baf2383a199cbc98648..a8d90e34b8d5a57b90eef9e383f2f99cbced952a 100644 --- a/checkpoints/metadata_000000753664.json +++ b/checkpoints/metadata_000000753664.json @@ -1 +1 @@ -{"step": 23, "tokens_seen": 753664, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.354241956204268} \ No newline at end of file +{"step": 23, "tokens_seen": 753664, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.457512075850191} \ No newline at end of file diff --git a/checkpoints/metadata_000000819200.json b/checkpoints/metadata_000000819200.json index 4f6d9fd3e1367ca7d97c9adc2a2d9f3dc48bab23..cba336037c82dcc45053f8107c9356ab3b31822a 100644 --- a/checkpoints/metadata_000000819200.json +++ b/checkpoints/metadata_000000819200.json @@ -1 +1 @@ -{"step": 25, "tokens_seen": 819200, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.337766838322462} \ No newline at end of file +{"step": 25, "tokens_seen": 819200, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.449279721179963} \ No newline at end of file diff --git a/checkpoints/metadata_000000917504.json b/checkpoints/metadata_000000917504.json index 9c5dfdc0b21d0ae2119aaaa2a01556013a166e69..a1e50128a2dd21a2575b816499ae2b4d42918e7f 100644 --- a/checkpoints/metadata_000000917504.json +++ b/checkpoints/metadata_000000917504.json @@ -1 +1 @@ -{"step": 28, "tokens_seen": 917504, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.311147664996973} \ No newline at end of file +{"step": 28, "tokens_seen": 917504, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.434475006481613} \ No newline at end of file diff --git a/checkpoints/metadata_000000983040.json b/checkpoints/metadata_000000983040.json index bc010ed384e2e1f4601abcdd8b9cae9018ae23b4..66670702a9e7ee685ed878d68411f7808b36849e 100644 --- a/checkpoints/metadata_000000983040.json +++ b/checkpoints/metadata_000000983040.json @@ -1 +1 @@ -{"step": 60, "tokens_seen": 983040, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 10.066979510761639} \ No newline at end of file +{"step": 30, "tokens_seen": 983040, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.423211635413145} \ No newline at end of file diff --git a/checkpoints/metadata_000001114112.json b/checkpoints/metadata_000001114112.json index 57d5c64836e446acfa2457130d2dd4a58e985699..ae6c5383cd9f201abbc5611179817b9b621775eb 100644 --- a/checkpoints/metadata_000001114112.json +++ b/checkpoints/metadata_000001114112.json @@ -1 +1 @@ -{"step": 34, "tokens_seen": 1114112, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.252617267525892} \ No newline at end of file +{"step": 34, "tokens_seen": 1114112, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.399444538393796} \ No newline at end of file diff --git a/checkpoints/metadata_000001212416.json b/checkpoints/metadata_000001212416.json index 50dd91ce1b207d52dba858f8f25a3e0c61461175..f210ccb2d6ac17de57825bee176dff70d7a196ca 100644 --- a/checkpoints/metadata_000001212416.json +++ b/checkpoints/metadata_000001212416.json @@ -1 +1 @@ -{"step": 37, "tokens_seen": 1212416, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.222297308724979} \ No newline at end of file +{"step": 37, "tokens_seen": 1212416, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.381340059139271} \ No newline at end of file diff --git a/checkpoints/metadata_000001343488.json b/checkpoints/metadata_000001343488.json index 11021f316cf9b1227eec13a706b68ec4b60366e1..44bdb8f50805d865186fdf449e845ab9e1b6e265 100644 --- a/checkpoints/metadata_000001343488.json +++ b/checkpoints/metadata_000001343488.json @@ -1 +1 @@ -{"step": 41, "tokens_seen": 1343488, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.177976434778357} \ No newline at end of file +{"step": 41, "tokens_seen": 1343488, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.356302876533771} \ No newline at end of file diff --git a/checkpoints/metadata_000001474560.json b/checkpoints/metadata_000001474560.json index 6fbe50438c39fbfc959c500325f8c1404342da90..16f105277c9e3e4c7cc9ecdd8421955e88187632 100644 --- a/checkpoints/metadata_000001474560.json +++ b/checkpoints/metadata_000001474560.json @@ -1 +1 @@ -{"step": 45, "tokens_seen": 1474560, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.127112486045279} \ No newline at end of file +{"step": 45, "tokens_seen": 1474560, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.328675141025588} \ No newline at end of file diff --git a/checkpoints/metadata_000001605632.json b/checkpoints/metadata_000001605632.json index 0410a231bba6404e7269bbc75ac0437c400def4a..d8fbe3c657f5e215594210cab7369d0413f64f69 100644 --- a/checkpoints/metadata_000001605632.json +++ b/checkpoints/metadata_000001605632.json @@ -1 +1 @@ -{"step": 49, "tokens_seen": 1605632, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.069664843206045} \ No newline at end of file +{"step": 49, "tokens_seen": 1605632, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.299211628700437} \ No newline at end of file diff --git a/checkpoints/metadata_000001769472.json b/checkpoints/metadata_000001769472.json index 7e13831a019324010dff72cd450a528109d6fe6a..a56519452756725492ef2039e2560e5031bd362a 100644 --- a/checkpoints/metadata_000001769472.json +++ b/checkpoints/metadata_000001769472.json @@ -1 +1 @@ -{"step": 54, "tokens_seen": 1769472, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 9.994979895512557} \ No newline at end of file +{"step": 54, "tokens_seen": 1769472, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.261818793867313} \ No newline at end of file diff --git a/checkpoints/metadata_000001966080.json b/checkpoints/metadata_000001966080.json index 85f1311fdd63b36cdd23404a555c87cadef8ff86..d4d551429b21f2c4035c094a7fe3ee2022900ef8 100644 --- a/checkpoints/metadata_000001966080.json +++ b/checkpoints/metadata_000001966080.json @@ -1 +1 @@ -{"step": 60, "tokens_seen": 1966080, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 9.894455992269652} \ No newline at end of file +{"step": 60, "tokens_seen": 1966080, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.213050824344208} \ No newline at end of file diff --git a/checkpoints/metadata_000002162688.json b/checkpoints/metadata_000002162688.json index 3fa3fb93a6d7e43c21febb540817f72a82e6573c..6d866ff7b75209be99100eaad3d50808713557e6 100644 --- a/checkpoints/metadata_000002162688.json +++ b/checkpoints/metadata_000002162688.json @@ -1 +1 @@ -{"step": 66, "tokens_seen": 2162688, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 9.787456261828652} \ No newline at end of file +{"step": 66, "tokens_seen": 2162688, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.161067744520894} \ No newline at end of file diff --git a/checkpoints/metadata_000002359296.json b/checkpoints/metadata_000002359296.json index ffbe35846f8e852de42f010f0dfae0d28c259cd0..05d9840ebe77623c8f77ed989b6a3458c5e28281 100644 --- a/checkpoints/metadata_000002359296.json +++ b/checkpoints/metadata_000002359296.json @@ -1 +1 @@ -{"step": 72, "tokens_seen": 2359296, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 9.671371978705555} \ No newline at end of file +{"step": 72, "tokens_seen": 2359296, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.104137801951891} \ No newline at end of file diff --git a/checkpoints/metadata_000002621440.json b/checkpoints/metadata_000002621440.json index 0fe7cb5720a24774ef20332c56721ac37ab2be2c..08cf367140935cbf0b403e4364028fe6c904c460 100644 --- a/checkpoints/metadata_000002621440.json +++ b/checkpoints/metadata_000002621440.json @@ -1 +1 @@ -{"step": 80, "tokens_seen": 2621440, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 9.511887299894651} \ No newline at end of file +{"step": 80, "tokens_seen": 2621440, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.0240865981226} \ No newline at end of file diff --git a/checkpoints/metadata_000002883584.json b/checkpoints/metadata_000002883584.json index a04181a875ef1f9547766fa23696bb8303abfb5e..cc6a08e30c990d0110207beaf7561775f243cdf0 100644 --- a/checkpoints/metadata_000002883584.json +++ b/checkpoints/metadata_000002883584.json @@ -1 +1 @@ -{"step": 88, "tokens_seen": 2883584, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 9.339867149283457} \ No newline at end of file +{"step": 88, "tokens_seen": 2883584, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 9.935583404823696} \ No newline at end of file diff --git a/checkpoints/metadata_000003178496.json b/checkpoints/metadata_000003178496.json index 0416f651639941889b0b3f890ce2969de7b3df7c..e2c235f3cd287b52e65ce3609ed4225e0543e641 100644 --- a/checkpoints/metadata_000003178496.json +++ b/checkpoints/metadata_000003178496.json @@ -1 +1 @@ -{"step": 97, "tokens_seen": 3178496, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 9.149718302323809} \ No newline at end of file +{"step": 97, "tokens_seen": 3178496, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 9.8319135957099} \ No newline at end of file diff --git a/checkpoints/metadata_000003473408.json b/checkpoints/metadata_000003473408.json index c011cb594f1e17cacf62b287c3ecce4a1a0fb3ec..1edd636fa3ab3c2a986e1f9a0d27d8436e15e9de 100644 --- a/checkpoints/metadata_000003473408.json +++ b/checkpoints/metadata_000003473408.json @@ -1 +1 @@ -{"step": 106, "tokens_seen": 3473408, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 8.954948132966997} \ No newline at end of file +{"step": 106, "tokens_seen": 3473408, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 9.718994025528087} \ No newline at end of file diff --git a/checkpoints/metadata_000003833856.json b/checkpoints/metadata_000003833856.json index 9c9d9f885dda98ab4af485828c1281546f6053a0..119de2ce134d3450118f51bd5cdf64077e12b638 100644 --- a/checkpoints/metadata_000003833856.json +++ b/checkpoints/metadata_000003833856.json @@ -1 +1 @@ -{"step": 117, "tokens_seen": 3833856, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 8.725915986013053} \ No newline at end of file +{"step": 117, "tokens_seen": 3833856, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 9.57546330601019} \ No newline at end of file diff --git a/checkpoints/metadata_000004227072.json b/checkpoints/metadata_000004227072.json index 57df490695fcd3033722792a4d504ea6fc3bc84a..72d88231a0a5c486e2fc0692bb34d41116a8f43b 100644 --- a/checkpoints/metadata_000004227072.json +++ b/checkpoints/metadata_000004227072.json @@ -1 +1 @@ -{"step": 129, "tokens_seen": 4227072, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 8.479047281676335} \ No newline at end of file +{"step": 129, "tokens_seen": 4227072, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 9.408515196645045} \ No newline at end of file diff --git a/checkpoints/metadata_000004653056.json b/checkpoints/metadata_000004653056.json index f117a66cc2b9c04b421a30280056659e1210fe03..3887c831c8f9fc4b9f0041cb04baff231e456c51 100644 --- a/checkpoints/metadata_000004653056.json +++ b/checkpoints/metadata_000004653056.json @@ -1 +1 @@ -{"step": 142, "tokens_seen": 4653056, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 8.226625281843564} \ No newline at end of file +{"step": 142, "tokens_seen": 4653056, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 9.23164245179911} \ No newline at end of file diff --git a/checkpoints/metadata_000005111808.json b/checkpoints/metadata_000005111808.json index 4ccabdfbc0bcf547a0b85fb5163cac9b76c6e6a7..b994e41b69cb291283edfb7d3a551bb64c99a36b 100644 --- a/checkpoints/metadata_000005111808.json +++ b/checkpoints/metadata_000005111808.json @@ -1 +1 @@ -{"step": 156, "tokens_seen": 5111808, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.982290411331379} \ No newline at end of file +{"step": 156, "tokens_seen": 5111808, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 9.057175241009995} \ No newline at end of file diff --git a/checkpoints/metadata_000005603328.json b/checkpoints/metadata_000005603328.json index 9e719c1e3966fa2470d517941b15f9ac9cec4a1c..6015f205c8a928cb2c2e97510dfd4a01ea2b4374 100644 --- a/checkpoints/metadata_000005603328.json +++ b/checkpoints/metadata_000005603328.json @@ -1 +1 @@ -{"step": 171, "tokens_seen": 5603328, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.735026211560928} \ No newline at end of file +{"step": 171, "tokens_seen": 5603328, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 8.882055448413595} \ No newline at end of file diff --git a/checkpoints/metadata_000006193152.json b/checkpoints/metadata_000006193152.json index dfb51e4314f076d9937d33253069939ad59bd49e..e95c2eedbd498578ab26a56cc7b1080a1e17628e 100644 --- a/checkpoints/metadata_000006193152.json +++ b/checkpoints/metadata_000006193152.json @@ -1 +1 @@ -{"step": 189, "tokens_seen": 6193152, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.4613181284979175} \ No newline at end of file +{"step": 189, "tokens_seen": 6193152, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 8.695822624511257} \ No newline at end of file diff --git a/checkpoints/metadata_000006782976.json b/checkpoints/metadata_000006782976.json index 4df54fd265b791c9b2a4e18a772b7f1c17a0c625..2eaac6fc46125b70a358b46a641ba2469a321bdc 100644 --- a/checkpoints/metadata_000006782976.json +++ b/checkpoints/metadata_000006782976.json @@ -1 +1 @@ -{"step": 207, "tokens_seen": 6782976, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.223525674157066} \ No newline at end of file +{"step": 207, "tokens_seen": 6782976, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 8.536231485948507} \ No newline at end of file diff --git a/checkpoints/metadata_000007471104.json b/checkpoints/metadata_000007471104.json index 1ce7276f9e14acbe631144abe1f5026a13e5ca08..21593d07e61cbe70c222585fd04b28110c5ab7b4 100644 --- a/checkpoints/metadata_000007471104.json +++ b/checkpoints/metadata_000007471104.json @@ -1 +1 @@ -{"step": 228, "tokens_seen": 7471104, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.968900871965506} \ No newline at end of file +{"step": 228, "tokens_seen": 7471104, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 8.364853085187022} \ No newline at end of file diff --git a/checkpoints/metadata_000008224768.json b/checkpoints/metadata_000008224768.json index 51bc68555c44296ecff283677dcf260a9966be38..aab44dd78d277f7092b45179e7d3d4dda6c29955 100644 --- a/checkpoints/metadata_000008224768.json +++ b/checkpoints/metadata_000008224768.json @@ -1 +1 @@ -{"step": 251, "tokens_seen": 8224768, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.737094952796088} \ No newline at end of file +{"step": 251, "tokens_seen": 8224768, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 8.20858914456734} \ No newline at end of file diff --git a/checkpoints/metadata_000009043968.json b/checkpoints/metadata_000009043968.json index 8b15afa7269f57227453e419d1946bc7889ac5f3..8b825affed9d7abe7c7132b69acc4d1b44cc1caa 100644 --- a/checkpoints/metadata_000009043968.json +++ b/checkpoints/metadata_000009043968.json @@ -1 +1 @@ -{"step": 276, "tokens_seen": 9043968, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.516684829899095} \ No newline at end of file +{"step": 276, "tokens_seen": 9043968, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 8.057145939779847} \ No newline at end of file diff --git a/checkpoints/metadata_000009961472.json b/checkpoints/metadata_000009961472.json index 2fc18b0894a373ee5ba0448128cb92ea28408fe4..ade0bba7cc15688b64fe5aa0eea4dda778a3e836 100644 --- a/checkpoints/metadata_000009961472.json +++ b/checkpoints/metadata_000009961472.json @@ -1 +1 @@ -{"step": 304, "tokens_seen": 9961472, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.302264189489683} \ No newline at end of file +{"step": 304, "tokens_seen": 9961472, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.9097705594246275} \ No newline at end of file diff --git a/checkpoints/metadata_000010944512.json b/checkpoints/metadata_000010944512.json index 73382cbab8b993e5e827acc6a06adca240397e80..6c082f42c540ce7fae2ef96a2d8b3bcec86a9e12 100644 --- a/checkpoints/metadata_000010944512.json +++ b/checkpoints/metadata_000010944512.json @@ -1 +1 @@ -{"step": 334, "tokens_seen": 10944512, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.113615961017287} \ No newline at end of file +{"step": 334, "tokens_seen": 10944512, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.779419562356378} \ No newline at end of file diff --git a/checkpoints/metadata_000012058624.json b/checkpoints/metadata_000012058624.json index 23aeb35d1c40a7f771a558f83ab7815010913c03..2705345d21ff3248e05a6c60400f1ebcb9056e8a 100644 --- a/checkpoints/metadata_000012058624.json +++ b/checkpoints/metadata_000012058624.json @@ -1 +1 @@ -{"step": 368, "tokens_seen": 12058624, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 5.930052665016961} \ No newline at end of file +{"step": 368, "tokens_seen": 12058624, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.656809406264156} \ No newline at end of file diff --git a/checkpoints/metadata_000013271040.json b/checkpoints/metadata_000013271040.json index c7d72a9c4773ed9d10f4456e22bf8c8ec54a61b6..635d6cede3a0b6312d5442e9f2e7158285db9ac5 100644 --- a/checkpoints/metadata_000013271040.json +++ b/checkpoints/metadata_000013271040.json @@ -1 +1 @@ -{"step": 405, "tokens_seen": 13271040, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 5.770665921846031} \ No newline at end of file +{"step": 405, "tokens_seen": 13271040, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.54616898967276} \ No newline at end of file diff --git a/checkpoints/metadata_000014581760.json b/checkpoints/metadata_000014581760.json index 8a7867c3cc7d1a7ad5a6f996ee85bcf08850af31..a4921bda410627d1f55e53bf1f05164c38620be6 100644 --- a/checkpoints/metadata_000014581760.json +++ b/checkpoints/metadata_000014581760.json @@ -1 +1 @@ -{"step": 445, "tokens_seen": 14581760, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 5.6236167282136575} \ No newline at end of file +{"step": 445, "tokens_seen": 14581760, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.444932654284786} \ No newline at end of file diff --git a/checkpoints/metadata_000016056320.json b/checkpoints/metadata_000016056320.json index fff05a9a0f5db92153070791c16b6c4055ba3e05..40bfdbeab37dc7d94c5efb020ec768352c5dc014 100644 --- a/checkpoints/metadata_000016056320.json +++ b/checkpoints/metadata_000016056320.json @@ -1 +1 @@ -{"step": 490, "tokens_seen": 16056320, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 5.488310843039606} \ No newline at end of file +{"step": 490, "tokens_seen": 16056320, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.351047587313347} \ No newline at end of file diff --git a/checkpoints/metadata_000016384000.json b/checkpoints/metadata_000016384000.json index 1e8d6ef2970999490a1325bdb23520a49a5fe9b3..04bc9ff9fbec937c54ce36db9b3a3d2363b6f45f 100644 --- a/checkpoints/metadata_000016384000.json +++ b/checkpoints/metadata_000016384000.json @@ -1 +1 @@ -{"step": 1000, "tokens_seen": 16384000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 5.2575255807637165} \ No newline at end of file +{"step": 500, "tokens_seen": 16384000, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.338734772199267} \ No newline at end of file diff --git a/checkpoints/metadata_000017661952.json b/checkpoints/metadata_000017661952.json index 705a11d50d72f0f369ecc50d1be60c4f3207630d..f9bd3c343df37b98569baac4c82a628b2d25a8d0 100644 --- a/checkpoints/metadata_000017661952.json +++ b/checkpoints/metadata_000017661952.json @@ -1 +1 @@ -{"step": 539, "tokens_seen": 17661952, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 5.367383792049406} \ No newline at end of file +{"step": 539, "tokens_seen": 17661952, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.270154893091622} \ No newline at end of file diff --git a/checkpoints/metadata_000019431424.json b/checkpoints/metadata_000019431424.json index c53b84f83540d25bb279302fe7e55ed1a5255942..4fabdc5bef6cfb99fa1a112154f7b7e2ac4f0cfc 100644 --- a/checkpoints/metadata_000019431424.json +++ b/checkpoints/metadata_000019431424.json @@ -1 +1 @@ -{"step": 593, "tokens_seen": 19431424, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 5.244713683265055} \ No newline at end of file +{"step": 593, "tokens_seen": 19431424, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.18724598299121} \ No newline at end of file diff --git a/checkpoints/metadata_000021364736.json b/checkpoints/metadata_000021364736.json index 9c8cce0080df86ceedd63bdf5bae60b2e6e761b7..daae74f7234c543eec23cfd3858e9ab2ffe8f5d6 100644 --- a/checkpoints/metadata_000021364736.json +++ b/checkpoints/metadata_000021364736.json @@ -1 +1 @@ -{"step": 652, "tokens_seen": 21364736, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 5.1562503005627045} \ No newline at end of file +{"step": 652, "tokens_seen": 21364736, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.126651679338602} \ No newline at end of file diff --git a/checkpoints/metadata_000023494656.json b/checkpoints/metadata_000023494656.json index 25b67647fdf92e0a67d8fc8a3089e13c3ba1d384..d8790f44fb4db5f0baaad92087e9e7e64cdf6c0f 100644 --- a/checkpoints/metadata_000023494656.json +++ b/checkpoints/metadata_000023494656.json @@ -1 +1 @@ -{"step": 717, "tokens_seen": 23494656, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 5.064713024898478} \ No newline at end of file +{"step": 717, "tokens_seen": 23494656, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.061925158726517} \ No newline at end of file diff --git a/checkpoints/metadata_000025853952.json b/checkpoints/metadata_000025853952.json index d587c512c2eb34701830f07ff2c98b7b20d2d1d3..62831adcd554f3239040007a2f77a65c09f23e77 100644 --- a/checkpoints/metadata_000025853952.json +++ b/checkpoints/metadata_000025853952.json @@ -1 +1 @@ -{"step": 789, "tokens_seen": 25853952, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 4.9753058592233055} \ No newline at end of file +{"step": 789, "tokens_seen": 25853952, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.993057823476354} \ No newline at end of file diff --git a/checkpoints/metadata_000028442624.json b/checkpoints/metadata_000028442624.json index d1acb34432bb13d5e58883e60ff0d79ea1e95926..cc58ead9f9fa3f727154390b1df39177081b1265 100644 --- a/checkpoints/metadata_000028442624.json +++ b/checkpoints/metadata_000028442624.json @@ -1 +1 @@ -{"step": 868, "tokens_seen": 28442624, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 4.881099863088409} \ No newline at end of file +{"step": 868, "tokens_seen": 28442624, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.930244985859927} \ No newline at end of file diff --git a/checkpoints/metadata_000031293440.json b/checkpoints/metadata_000031293440.json index d88658860d777eed248a36bdf2c36d5af353b5c7..96026ce29ee5ea367b04903f43eb081d26cf81a7 100644 --- a/checkpoints/metadata_000031293440.json +++ b/checkpoints/metadata_000031293440.json @@ -1 +1 @@ -{"step": 955, "tokens_seen": 31293440, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 4.794771261136894} \ No newline at end of file +{"step": 955, "tokens_seen": 31293440, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.871021630989001} \ No newline at end of file diff --git a/checkpoints/metadata_000032768000.json b/checkpoints/metadata_000032768000.json index 354e8a52d984dce33dfa2a43303cc014cb2c4783..ab34743462b527e556e0a3afab28d8dbb64a4d3f 100644 --- a/checkpoints/metadata_000032768000.json +++ b/checkpoints/metadata_000032768000.json @@ -1 +1 @@ -{"step": 2000, "tokens_seen": 32768000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 4.695193499554638} \ No newline at end of file +{"step": 1000, "tokens_seen": 32768000, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.8403029303212} \ No newline at end of file diff --git a/checkpoints/metadata_000034439168.json b/checkpoints/metadata_000034439168.json index a8e3799243490be051c79701a4f9acfb66e1a31a..7e24bf605db9562684a9ed430ed586ce4523f0ca 100644 --- a/checkpoints/metadata_000034439168.json +++ b/checkpoints/metadata_000034439168.json @@ -1 +1 @@ -{"step": 1051, "tokens_seen": 34439168, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 4.687631394674228} \ No newline at end of file +{"step": 1051, "tokens_seen": 34439168, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.809880089493139} \ No newline at end of file diff --git a/checkpoints/metadata_000037879808.json b/checkpoints/metadata_000037879808.json index 1c5cc412b5f4989330e74177912095f348fa7dca..39ec96a088382f668b90c28179c2452066579241 100644 --- a/checkpoints/metadata_000037879808.json +++ b/checkpoints/metadata_000037879808.json @@ -1 +1 @@ -{"step": 1156, "tokens_seen": 37879808, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 4.584697024859014} \ No newline at end of file +{"step": 1156, "tokens_seen": 37879808, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.752295873041866} \ No newline at end of file diff --git a/checkpoints/metadata_000041648128.json b/checkpoints/metadata_000041648128.json index a32ad5d552756103c7607aaeb761686e767b19f6..17eb3c0009a5cdb133e8866941dfe4e7b6e0eb4f 100644 --- a/checkpoints/metadata_000041648128.json +++ b/checkpoints/metadata_000041648128.json @@ -1 +1 @@ -{"step": 1271, "tokens_seen": 41648128, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 4.479492940101269} \ No newline at end of file +{"step": 1271, "tokens_seen": 41648128, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.6895263577832145} \ No newline at end of file diff --git a/checkpoints/metadata_000045842432.json b/checkpoints/metadata_000045842432.json index 8c4e52a49c82fe868b9df64d434ea021e89d11ab..7e12e99dab4c655ebd8465a26c5bbc277f01c4fd 100644 --- a/checkpoints/metadata_000045842432.json +++ b/checkpoints/metadata_000045842432.json @@ -1 +1 @@ -{"step": 1399, "tokens_seen": 45842432, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 4.359371060873147} \ No newline at end of file +{"step": 1399, "tokens_seen": 45842432, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.63130592900971} \ No newline at end of file diff --git a/checkpoints/metadata_000049152000.json b/checkpoints/metadata_000049152000.json index 9c14a98683104a146a53093fc45286cfe5abf78f..10b24aecb0445274861eefe5f2ec5d46c8679212 100644 --- a/checkpoints/metadata_000049152000.json +++ b/checkpoints/metadata_000049152000.json @@ -1 +1 @@ -{"step": 3000, "tokens_seen": 49152000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 4.24904853940907} \ No newline at end of file +{"step": 1500, "tokens_seen": 49152000, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.593775578711349} \ No newline at end of file diff --git a/checkpoints/metadata_000050397184.json b/checkpoints/metadata_000050397184.json index 47a435ae122d752ec16089b4ab3efe1f3218b2af..1d28a1b1054b6183821a1bf8da3e35a7d7008ae2 100644 --- a/checkpoints/metadata_000050397184.json +++ b/checkpoints/metadata_000050397184.json @@ -1 +1 @@ -{"step": 1538, "tokens_seen": 50397184, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 4.284214765378866} \ No newline at end of file +{"step": 1538, "tokens_seen": 50397184, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.589469263522446} \ No newline at end of file diff --git a/checkpoints/metadata_000055443456.json b/checkpoints/metadata_000055443456.json index 8540b414d45531f7f9611373b383f8a0da2d40c9..48210befed102b21dce1046d27b2101db1c733ba 100644 --- a/checkpoints/metadata_000055443456.json +++ b/checkpoints/metadata_000055443456.json @@ -1 +1 @@ -{"step": 1692, "tokens_seen": 55443456, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 4.135964355594053} \ No newline at end of file +{"step": 1692, "tokens_seen": 55443456, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.515140544254375} \ No newline at end of file diff --git a/checkpoints/metadata_000061014016.json b/checkpoints/metadata_000061014016.json index 45cda348df93d678d78192798e53c542accc4377..3c6f435beb914c273ac3fec145494ed0e551a76c 100644 --- a/checkpoints/metadata_000061014016.json +++ b/checkpoints/metadata_000061014016.json @@ -1 +1 @@ -{"step": 1862, "tokens_seen": 61014016, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 3.9819966090161265} \ No newline at end of file +{"step": 1862, "tokens_seen": 61014016, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.468975151474897} \ No newline at end of file diff --git a/checkpoints/metadata_000065536000.json b/checkpoints/metadata_000065536000.json index fae4e50bc9ce9dab67b6646f8bb3bf9e599b9635..30ed54c65badcb85ad1bdbb37562fed400032860 100644 --- a/checkpoints/metadata_000065536000.json +++ b/checkpoints/metadata_000065536000.json @@ -1 +1 @@ -{"step": 4000, "tokens_seen": 65536000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.805410613469591} \ No newline at end of file +{"step": 2000, "tokens_seen": 65536000, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.455209961214881} \ No newline at end of file diff --git a/checkpoints/metadata_000067108864.json b/checkpoints/metadata_000067108864.json index 174a8a4c34c5aa7bb990c312446467d2ab662437..a3262b40af09de49797b6d825937d41e4b067f1a 100644 --- a/checkpoints/metadata_000067108864.json +++ b/checkpoints/metadata_000067108864.json @@ -1 +1 @@ -{"step": 2048, "tokens_seen": 67108864, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 3.852328401139968} \ No newline at end of file +{"step": 2048, "tokens_seen": 67108864, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.442047359333899} \ No newline at end of file diff --git a/checkpoints/metadata_000073826304.json b/checkpoints/metadata_000073826304.json index 9c710b3aa9fc103bf412d359ba2f9ed6a3bbce9c..e8d877831200fc7998001d402272e4d07d059db1 100644 --- a/checkpoints/metadata_000073826304.json +++ b/checkpoints/metadata_000073826304.json @@ -1 +1 @@ -{"step": 2253, "tokens_seen": 73826304, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 3.70811647117009} \ No newline at end of file +{"step": 2253, "tokens_seen": 73826304, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.392868120519023} \ No newline at end of file diff --git a/checkpoints/metadata_000081199104.json b/checkpoints/metadata_000081199104.json index 3b64cc8dee28aca55271df982d9d5ea90b3c1e4d..e4df7cf0c0972901ab42494f458b6e5b907cfa54 100644 --- a/checkpoints/metadata_000081199104.json +++ b/checkpoints/metadata_000081199104.json @@ -1 +1 @@ -{"step": 2478, "tokens_seen": 81199104, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 3.616063727550929} \ No newline at end of file +{"step": 2478, "tokens_seen": 81199104, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.3576589838192445} \ No newline at end of file diff --git a/checkpoints/metadata_000081920000.json b/checkpoints/metadata_000081920000.json index cad9a7d74f8b17ce1ae1a2a6436c22cff2e42cb5..5d7dd37e3f49e0cc65e236873a7dc278d299d6f2 100644 --- a/checkpoints/metadata_000081920000.json +++ b/checkpoints/metadata_000081920000.json @@ -1 +1 @@ -{"step": 5000, "tokens_seen": 81920000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.556630901397966} \ No newline at end of file +{"step": 2500, "tokens_seen": 81920000, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.350992743038883} \ No newline at end of file diff --git a/checkpoints/metadata_000089325568.json b/checkpoints/metadata_000089325568.json index c31353c6d44f39edfe38802d6565a7dbd6ed4822..073b23616ea2c5af54c0ba1a8a5412e816dec4db 100644 --- a/checkpoints/metadata_000089325568.json +++ b/checkpoints/metadata_000089325568.json @@ -1 +1 @@ -{"step": 2726, "tokens_seen": 89325568, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 3.5297980905706705} \ No newline at end of file +{"step": 2726, "tokens_seen": 89325568, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.3118517179542515} \ No newline at end of file diff --git a/checkpoints/metadata_000098271232.json b/checkpoints/metadata_000098271232.json index 8efc947e08df077ea64c6618c46f57fc06c1e7fe..d2285571945334db35926d7312782ae384aab733 100644 --- a/checkpoints/metadata_000098271232.json +++ b/checkpoints/metadata_000098271232.json @@ -1 +1 @@ -{"step": 2999, "tokens_seen": 98271232, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 3.4688791846861817} \ No newline at end of file +{"step": 2999, "tokens_seen": 98271232, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.282990001006669} \ No newline at end of file diff --git a/checkpoints/metadata_000098304000.json b/checkpoints/metadata_000098304000.json index 0056bd2f5175921a34492a8581779895036ccd93..7831d79b67840e8da5726d2c94d05530910d6a5e 100644 --- a/checkpoints/metadata_000098304000.json +++ b/checkpoints/metadata_000098304000.json @@ -1 +1 @@ -{"step": 6000, "tokens_seen": 98304000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.448754568027084} \ No newline at end of file +{"step": 3000, "tokens_seen": 98304000, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.282419999182332} \ No newline at end of file diff --git a/checkpoints/metadata_000108068864.json b/checkpoints/metadata_000108068864.json index 71baf7c60f0aeebf7499a5b32f43df43aff6faff..f96caf67de8233ad48d0af16c9c61de581bb8084 100644 --- a/checkpoints/metadata_000108068864.json +++ b/checkpoints/metadata_000108068864.json @@ -1 +1 @@ -{"step": 3298, "tokens_seen": 108068864, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 3.402881423257483} \ No newline at end of file +{"step": 3298, "tokens_seen": 108068864, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.241222084404356} \ No newline at end of file diff --git a/checkpoints/metadata_000114688000.json b/checkpoints/metadata_000114688000.json index 12fc826592c2e716014285f77cc2dc57f87ccd00..74aa2f40009687176c2d890c22b7e4267dd817b7 100644 --- a/checkpoints/metadata_000114688000.json +++ b/checkpoints/metadata_000114688000.json @@ -1 +1 @@ -{"step": 7000, "tokens_seen": 114688000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.332638167603121} \ No newline at end of file +{"step": 3500, "tokens_seen": 114688000, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.215461925557858} \ No newline at end of file diff --git a/checkpoints/metadata_000118882304.json b/checkpoints/metadata_000118882304.json index 4bb20bd7dcb247af15e3be802decd0ec9d567e95..1a461804332d20fd28a58f66cb6fcc8948c7af6c 100644 --- a/checkpoints/metadata_000118882304.json +++ b/checkpoints/metadata_000118882304.json @@ -1 +1 @@ -{"step": 3628, "tokens_seen": 118882304, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 3.363343283490335} \ No newline at end of file +{"step": 3628, "tokens_seen": 118882304, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.217147022955933} \ No newline at end of file diff --git a/checkpoints/metadata_000130777088.json b/checkpoints/metadata_000130777088.json index 80a21213c61b3e6ee6a2123bfa342cdfffeb47f1..55ee41b69ac4234da40ae271eba6cd6608876cae 100644 --- a/checkpoints/metadata_000130777088.json +++ b/checkpoints/metadata_000130777088.json @@ -1 +1 @@ -{"step": 3991, "tokens_seen": 130777088, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 3.315968705398669} \ No newline at end of file +{"step": 3991, "tokens_seen": 130777088, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.181719852071981} \ No newline at end of file diff --git a/checkpoints/metadata_000131072000.json b/checkpoints/metadata_000131072000.json index aa3bbd65fdb0095554bf3137360076d85e799386..34fa071125bce831c47d78940830975df18df086 100644 --- a/checkpoints/metadata_000131072000.json +++ b/checkpoints/metadata_000131072000.json @@ -1 +1 @@ -{"step": 8000, "tokens_seen": 131072000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.309278600746995} \ No newline at end of file +{"step": 4000, "tokens_seen": 131072000, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.185208025248347} \ No newline at end of file diff --git a/checkpoints/metadata_000143851520.json b/checkpoints/metadata_000143851520.json index 5b366520139f62f32557566f19839bcea1067bda..e14dd4c51dde13fde103e3aeeeeaa64e188366f2 100644 --- a/checkpoints/metadata_000143851520.json +++ b/checkpoints/metadata_000143851520.json @@ -1 +1 @@ -{"step": 4390, "tokens_seen": 143851520, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 3.2712891613103365} \ No newline at end of file +{"step": 4390, "tokens_seen": 143851520, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.15403079619963} \ No newline at end of file diff --git a/checkpoints/metadata_000147456000.json b/checkpoints/metadata_000147456000.json index ae36f700a1aae921262a1b2c42ba31e9c2daaa6d..c38f729b500b85502e99d0930e2097c66eeba48a 100644 --- a/checkpoints/metadata_000147456000.json +++ b/checkpoints/metadata_000147456000.json @@ -1 +1 @@ -{"step": 9000, "tokens_seen": 147456000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.234525672614021} \ No newline at end of file +{"step": 4500, "tokens_seen": 147456000, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.149331373865662} \ No newline at end of file diff --git a/checkpoints/metadata_000158269440.json b/checkpoints/metadata_000158269440.json index 920abcd8c3277ee5a50bfa8d98afb6fa40252227..552cbdc282ad73f8102163af94a026a62a98f7fe 100644 --- a/checkpoints/metadata_000158269440.json +++ b/checkpoints/metadata_000158269440.json @@ -1 +1 @@ -{"step": 4830, "tokens_seen": 158269440, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 3.2387536035281825} \ No newline at end of file +{"step": 4830, "tokens_seen": 158269440, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.127522719823752} \ No newline at end of file diff --git a/checkpoints/metadata_000163840000.json b/checkpoints/metadata_000163840000.json index 846316fa8560931e4e606fedbfb982b2dd8537a4..8cf82a14e4a082e5da5ad51ca652bfeceb472431 100644 --- a/checkpoints/metadata_000163840000.json +++ b/checkpoints/metadata_000163840000.json @@ -1 +1 @@ -{"step": 10000, "tokens_seen": 163840000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.199526206936364} \ No newline at end of file +{"step": 5000, "tokens_seen": 163840000, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.108985160469056} \ No newline at end of file diff --git a/checkpoints/metadata_000174096384.json b/checkpoints/metadata_000174096384.json index e75608397d95e7ce870a175acbcf45d64df70d7f..767f65685d4a653988e308701d34ff6a85dc6567 100644 --- a/checkpoints/metadata_000174096384.json +++ b/checkpoints/metadata_000174096384.json @@ -1 +1 @@ -{"step": 5313, "tokens_seen": 174096384, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 3.2087791353150585} \ No newline at end of file +{"step": 5313, "tokens_seen": 174096384, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.09481939208637} \ No newline at end of file diff --git a/checkpoints/metadata_000180224000.json b/checkpoints/metadata_000180224000.json index b2916f82e5dcc3f6fc98307c8e77a71dccdedfcb..8839136df4b03f81e8853fd18be1edea1555722f 100644 --- a/checkpoints/metadata_000180224000.json +++ b/checkpoints/metadata_000180224000.json @@ -1 +1 @@ -{"step": 11000, "tokens_seen": 180224000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.1686492056920414} \ No newline at end of file +{"step": 5500, "tokens_seen": 180224000, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.078852562646926} \ No newline at end of file diff --git a/checkpoints/metadata_000191496192.json b/checkpoints/metadata_000191496192.json index 48e0ea61ef492a89ba4df49cbf9564391262e98e..2c5a2f866630c346313ece03d43ad01382da1c56 100644 --- a/checkpoints/metadata_000191496192.json +++ b/checkpoints/metadata_000191496192.json @@ -1 +1 @@ -{"step": 5844, "tokens_seen": 191496192, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 3.1639664520238506} \ No newline at end of file +{"step": 5844, "tokens_seen": 191496192, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.06682519208268} \ No newline at end of file diff --git a/checkpoints/metadata_000196608000.json b/checkpoints/metadata_000196608000.json index 723ca80bda508b65d1a5a9ba54d02aa5b4ff38dc..3008eaea9c3c83cb3eeec580a0e58f5d6387c22d 100644 --- a/checkpoints/metadata_000196608000.json +++ b/checkpoints/metadata_000196608000.json @@ -1 +1 @@ -{"step": 12000, "tokens_seen": 196608000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.138256716618341} \ No newline at end of file +{"step": 6000, "tokens_seen": 196608000, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.055781584540544} \ No newline at end of file diff --git a/checkpoints/metadata_000196706304.json b/checkpoints/metadata_000196706304.json index 091ba727afb589129184a806c1cab1fc05faeb3d..55c8cecd9e130977dda4b7da50e6b721d54cda40 100644 --- a/checkpoints/metadata_000196706304.json +++ b/checkpoints/metadata_000196706304.json @@ -1 +1 @@ -{"step": 6003, "tokens_seen": 196706304, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 3.170731699051464} \ No newline at end of file +{"step": 6003, "tokens_seen": 196706304, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.057905402510601} \ No newline at end of file diff --git a/checkpoints/metadata_000197361664.json b/checkpoints/metadata_000197361664.json index 4dd976c60671fb365de6d65d072afa3398c65704..30c87428cf1265c97780faa6be8e22c2eab44efa 100644 --- a/checkpoints/metadata_000197361664.json +++ b/checkpoints/metadata_000197361664.json @@ -1 +1 @@ -{"step": 6023, "tokens_seen": 197361664, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 3.171016553932927} \ No newline at end of file +{"step": 6023, "tokens_seen": 197361664, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.0602384537326} \ No newline at end of file diff --git a/checkpoints/metadata_000198017024.json b/checkpoints/metadata_000198017024.json index 38cdb48c0dd05bf9a39895dc6e72069290375bd7..c6f0b31206a1ffcceb72ab5e0ac89881ddf2a8cd 100644 --- a/checkpoints/metadata_000198017024.json +++ b/checkpoints/metadata_000198017024.json @@ -1 +1 @@ -{"step": 6043, "tokens_seen": 198017024, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 3.169122244180558} \ No newline at end of file +{"step": 6043, "tokens_seen": 198017024, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.056899222229194} \ No newline at end of file diff --git a/checkpoints/metadata_000198672384.json b/checkpoints/metadata_000198672384.json index 3c8e8b4c2235fb166c9c118d7b1c647de04111e9..f4b196b8c7d737cadba1c44bbac9b54dcfba81ec 100644 --- a/checkpoints/metadata_000198672384.json +++ b/checkpoints/metadata_000198672384.json @@ -1 +1 @@ -{"step": 6063, "tokens_seen": 198672384, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 3.157218086042496} \ No newline at end of file +{"step": 6063, "tokens_seen": 198672384, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.0501680024358855} \ No newline at end of file diff --git a/checkpoints/metadata_000199327744.json b/checkpoints/metadata_000199327744.json index 36a32e7174c63e3f9e67c6ea5a6b75a14a018aa4..f04a2ba7fe20289d3d54b45fe868fca2ef0f413a 100644 --- a/checkpoints/metadata_000199327744.json +++ b/checkpoints/metadata_000199327744.json @@ -1 +1 @@ -{"step": 6083, "tokens_seen": 199327744, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 3.162601982560343} \ No newline at end of file +{"step": 6083, "tokens_seen": 199327744, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.055669312745273} \ No newline at end of file diff --git a/checkpoints/metadata_000199950336.json b/checkpoints/metadata_000199950336.json index 389eb4b3f672e2d2c62d2b59ee77e494577e09db..fb90620e84e611d9fb437aadda268c8c2a26688f 100644 --- a/checkpoints/metadata_000199950336.json +++ b/checkpoints/metadata_000199950336.json @@ -1 +1 @@ -{"step": 6102, "tokens_seen": 199950336, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 3.1541889009453166} \ No newline at end of file +{"step": 6102, "tokens_seen": 199950336, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.048876639155322} \ No newline at end of file diff --git a/checkpoints/model_weights_000000032768.pt b/checkpoints/model_weights_000000032768.pt index 7f6221a777783bc8bde50d7335a40476671f01ca..b078b2917bda543b2dcc8cbf674e7870d98b0346 100644 --- a/checkpoints/model_weights_000000032768.pt +++ b/checkpoints/model_weights_000000032768.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:89ab8b7a75e0611edbb0998131098c13642cfcb21a4b84411f8557a8dfb3f4d3 -size 234226143 +oid sha256:db649a536cb0a57b27c8d60c5bb440a77954c7b96613bd240a49eda4fca2fb84 +size 158534613 diff --git a/checkpoints/model_weights_000000327680.pt b/checkpoints/model_weights_000000327680.pt index f6bd4628bb2bc97fcfd18ccd18b71bcba01da25f..dffe527ccc62c9143dd65aa7b59a258c95bb365b 100644 --- a/checkpoints/model_weights_000000327680.pt +++ b/checkpoints/model_weights_000000327680.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e919b6c40b2489584d343c4750166faa04ad1deaa6de0ddd9ed60268d9e1e6ec -size 234226143 +oid sha256:46f0baa20a6093d3946367cf7689267292f54fd7569c6d1d11000d9f5c1fd0d2 +size 158534613 diff --git a/checkpoints/model_weights_000000360448.pt b/checkpoints/model_weights_000000360448.pt index 902b4bc3b41770af8a4a94670079dc631e46ef4b..55ecd84d037534d5346969ba0e6fd935335ea020 100644 --- a/checkpoints/model_weights_000000360448.pt +++ b/checkpoints/model_weights_000000360448.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:542473aff0cb1db66f02bc9bbe4dc366161165ccb834fa51c6d6752ddeb1438f -size 234226143 +oid sha256:3d1c061e9184336f407409131645ad11de00d6a628dcb74655b84e594533310b +size 158534613 diff --git a/checkpoints/model_weights_000000425984.pt b/checkpoints/model_weights_000000425984.pt index d3d6948415f6beb6751ceaa41f128b86cc602477..e04750a1cfd036e07e4a72676e3c34a7985f2e09 100644 --- a/checkpoints/model_weights_000000425984.pt +++ b/checkpoints/model_weights_000000425984.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:dc8ce9c839e1cfc5ae837e14f9f44230bda4df49491ae6664dd8691f30f9cec1 -size 234226143 +oid sha256:a5904405e7d013e4b9bb7154825ad7138449cb629731c75577010a0f2411b15f +size 158534613 diff --git a/checkpoints/model_weights_000000458752.pt b/checkpoints/model_weights_000000458752.pt index 746bd3ccaf3d69b43ee3c2be6a2fc1855571a63e..4888aed92d31b0a648357a00b3519bd9e58ccad1 100644 --- a/checkpoints/model_weights_000000458752.pt +++ b/checkpoints/model_weights_000000458752.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:14c21a764eef3ea325226ec63f9f33dae1df0ef038a0a14d40fc8227ad168479 -size 284687203 +oid sha256:98226a69ca50af09f3ce4135aca426c1d9d0fea99d5080f8ff75e4da25a9ec6c +size 158534613 diff --git a/checkpoints/model_weights_000000491520.pt b/checkpoints/model_weights_000000491520.pt index c9fcda64d5a9be27a93ccb8a40489c57bfe10880..ede2d3aa6677727467732e1f61ba28491a344e80 100644 --- a/checkpoints/model_weights_000000491520.pt +++ b/checkpoints/model_weights_000000491520.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0f3b5321be86eefe9a862a3cb2f5ead22656f3a1160424c20bdb59e4808404e5 -size 284687203 +oid sha256:0ce4df85e2f5f4c0f22200320da8963e082870b7cffb057d70f14740922bd6de +size 158534613 diff --git a/checkpoints/model_weights_000000557056.pt b/checkpoints/model_weights_000000557056.pt index 7b6bfe1a5940ae983e3a4a5cbfaa3af89cefa1d0..dc56213965caeb331b04c26a6ec3dc9218ff7c32 100644 --- a/checkpoints/model_weights_000000557056.pt +++ b/checkpoints/model_weights_000000557056.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:41cd7c2367161f15614475bddf6bcf5b766ee7629b48d634017c1c7e6bd38674 -size 284687203 +oid sha256:15b184381a3b5a2f681399cd75043bed4cc8299603cb5865f426eaeabf421478 +size 158534613 diff --git a/checkpoints/model_weights_000000622592.pt b/checkpoints/model_weights_000000622592.pt index 2c4dcd13b67e2d9bfce6193bc52053049c6f6f5c..bf28481a5e9a7b88ec1bf823145efab4b7155777 100644 --- a/checkpoints/model_weights_000000622592.pt +++ b/checkpoints/model_weights_000000622592.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d87ec2e8370d2d8b83941c4a2bd0be914b2d3eb75a7b154605958eda3540e1a8 -size 234226143 +oid sha256:786b7528e4e26f7ebcf2b6ff451991ae4938c5c22e81171d818e6589ebd26804 +size 158534613 diff --git a/checkpoints/model_weights_000000688128.pt b/checkpoints/model_weights_000000688128.pt index fa1ffaba69669307124a99a6d82895a1fc196c0d..a368a32755805386c7e3f94e2a233e495d7a0144 100644 --- a/checkpoints/model_weights_000000688128.pt +++ b/checkpoints/model_weights_000000688128.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:96e2432cbcc9ca950d50b6aef0a6f10b7a3665feea562e207b951bda43dcf06f -size 234226143 +oid sha256:477dc2959ac960b82876582c909977641ffc0206d48aa5d1c84a0b818d7f5244 +size 158534613 diff --git a/checkpoints/model_weights_000000753664.pt b/checkpoints/model_weights_000000753664.pt index 1f7e79d68d09ffe329f1ee896630e723569880c0..ba9b0780b0414c4bd60b017b2005b7e5756bd0f0 100644 --- a/checkpoints/model_weights_000000753664.pt +++ b/checkpoints/model_weights_000000753664.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ddc97a8e9a1935a85072e4c3311362b6174d98c22ba0d0bc8f5ae94b3e18b419 -size 234226143 +oid sha256:769ca8c1dd1f30ab40cc13e263c94210a892aa44bf0f7ed638405f4bd7424ddc +size 158534613 diff --git a/checkpoints/model_weights_000000819200.pt b/checkpoints/model_weights_000000819200.pt index b90d53654b4c1a75848b339edaf88eb3d18d0a5e..6cc80b06e3aa99aace504249127b19d3f29de72e 100644 --- a/checkpoints/model_weights_000000819200.pt +++ b/checkpoints/model_weights_000000819200.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:da3f925a6a633ace6fd69547852513754b71ddc64a6aaecfd8b9caf99975b9e4 -size 234226143 +oid sha256:4ebe62b17b28b4b321549d3291a2f74fec115af635c88150dc07ebc835a42978 +size 158534613 diff --git a/checkpoints/model_weights_000000917504.pt b/checkpoints/model_weights_000000917504.pt index ccfd6d4b61e46b6231541a93822d56bb6eb7735d..4cc963c1abc255d6819efc9fa70efeeb6df6f19a 100644 --- a/checkpoints/model_weights_000000917504.pt +++ b/checkpoints/model_weights_000000917504.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:07a8f2fbe1e8fe3ff169c64d240a8d523bca992961447052b98ebf6df9c72ba0 -size 234226143 +oid sha256:be38c151e87f18846fa2b59d9856cb93b4d5d875568cc80689a047d6ec1499f7 +size 158534613 diff --git a/checkpoints/model_weights_000000983040.pt b/checkpoints/model_weights_000000983040.pt index d4747a7df432b00dc21d3d4ad11ef5b8995f0c74..d92782acd30054a679e1910517b8aba2a2217fb7 100644 --- a/checkpoints/model_weights_000000983040.pt +++ b/checkpoints/model_weights_000000983040.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:44c3ce79770dd2a3bd2b78484b0259178f03d92c7c3227f32927a22160b0a665 -size 284687203 +oid sha256:56cfd512a5e1054654e9a612b5536023145829ff1c95e5049f486196f0800f44 +size 158534613 diff --git a/checkpoints/model_weights_000001114112.pt b/checkpoints/model_weights_000001114112.pt index 0686facd020770139cd2b38460493b97ec182962..4ae63f75e13048c28fe4454ceab9b6e13cf1fd8e 100644 --- a/checkpoints/model_weights_000001114112.pt +++ b/checkpoints/model_weights_000001114112.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e83b6a781bb4390bc10296470f7482c5cb90f75c80482146d306cccda267af6b -size 234226143 +oid sha256:7754b643840b1f56735e654c6b778300b639b40e7e30beec3556451e322f570b +size 158534613 diff --git a/checkpoints/model_weights_000001212416.pt b/checkpoints/model_weights_000001212416.pt index 75df8d232d29c64925764e6f77e3d87e94dc3d5a..b82edface0784bba5610abf90495262506b280e1 100644 --- a/checkpoints/model_weights_000001212416.pt +++ b/checkpoints/model_weights_000001212416.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:37bfab82766373124e39cb3eec839fcd69c312465aa6f5be15be972a9471e1aa -size 234226143 +oid sha256:c5494efcdfe1a51ddbf158a24a147ed569333a5da087ace1c10e3918ffa14959 +size 158534613 diff --git a/checkpoints/model_weights_000001343488.pt b/checkpoints/model_weights_000001343488.pt index bbaa9b3374c698d41479aa298614a9ab83767d9a..1d492b8687aa871100a6c0b61b9b792cefa3c615 100644 --- a/checkpoints/model_weights_000001343488.pt +++ b/checkpoints/model_weights_000001343488.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:477bd84800c8b4dafc151d62ccfc7bb07cddfc82d5d778a60cc920ec31b5410c -size 234226143 +oid sha256:96aafbdd743f66bf65d03b3a433e4b12bdac27ebacda2e04b3b66cab1fdffbe6 +size 158534613 diff --git a/checkpoints/model_weights_000001474560.pt b/checkpoints/model_weights_000001474560.pt index 8c050fbff9e32c1ecfcad01172e74d326bd79938..60d62b452fa7bb08f5a49d580ce590b079881e56 100644 --- a/checkpoints/model_weights_000001474560.pt +++ b/checkpoints/model_weights_000001474560.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4346551f028b9f10c6d4523c4619026ecef9242b1ebcf98d555df150f5aee1cf -size 234226143 +oid sha256:68da71520499e173eb71e0d15c51556d95d6b0096f8daa54e6a1374613cae55e +size 158534613 diff --git a/checkpoints/model_weights_000001605632.pt b/checkpoints/model_weights_000001605632.pt index 68a4dc162c9521a7da9b6f6f1a5911ce21d715e7..66cc0d29e38823f68f05b404bd60455ec5a6acdb 100644 --- a/checkpoints/model_weights_000001605632.pt +++ b/checkpoints/model_weights_000001605632.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8c7bb6ff56f7dc2f7a24587ca39650d1d9ceb0ac13a8a6360181b20fffc7bff2 -size 234226143 +oid sha256:4cb46bd9cc7ad3b47422287245842ca6d8ec2959e1af10a34814a2d8eee8b460 +size 158534613 diff --git a/checkpoints/model_weights_000001769472.pt b/checkpoints/model_weights_000001769472.pt index 7feeefb7da097222e3ada9ac704d3707ee804eb7..038978cdec431a0a7df222a1ccfd29f9befd377a 100644 --- a/checkpoints/model_weights_000001769472.pt +++ b/checkpoints/model_weights_000001769472.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2d82f842ee0ee91e9cf6514d719c7cdc8126fa137bc66554832dfd8ac60f2748 -size 234226143 +oid sha256:869cd0361c5303a7a37c8f9a77d25fae0bb26873592af6649857b22e07f7e3b4 +size 158534613 diff --git a/checkpoints/model_weights_000001966080.pt b/checkpoints/model_weights_000001966080.pt index 430ef6d253666c5acc8375746060c3d809c286c1..b371a5f8926bd612f264422b77b106440e2e7974 100644 --- a/checkpoints/model_weights_000001966080.pt +++ b/checkpoints/model_weights_000001966080.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:596b1ab35a23ff5183b566e1ed0c7cf07c8809894261a007b517a125bfc45d94 -size 234226143 +oid sha256:2cb82221f57fe958d468f082aed228df5f14ed663c1a4651942370248a34cfdd +size 158534613 diff --git a/checkpoints/model_weights_000002162688.pt b/checkpoints/model_weights_000002162688.pt index feb22aeb0517993cd3e73a22a8ed7b445f03ed79..56322536d83826b1b3e96651d146bd218faeb1ee 100644 --- a/checkpoints/model_weights_000002162688.pt +++ b/checkpoints/model_weights_000002162688.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:beb251cf79705d29cf2da2f32037e82b14ec2c2f2ec1ee88ea37b27effa11fcf -size 234226143 +oid sha256:49ea20f33fcb0708ff3be13175c332398d05631c9f381dd28485945fcdf4b13e +size 158534613 diff --git a/checkpoints/model_weights_000002359296.pt b/checkpoints/model_weights_000002359296.pt index 698dbb638292ef8338130b0f5d71a02200a3756e..5adb3f844d648cd8a342d2f18f37cbdb4854f6dc 100644 --- a/checkpoints/model_weights_000002359296.pt +++ b/checkpoints/model_weights_000002359296.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:583a1c68a6520f7c6edb53c1da582a829b79921fa43694e848d1532fca7bd9ae -size 234226143 +oid sha256:965e39fe5a1276b8394a1030db161c3d12cb29b253ff5d2dd98ad6f8c3edc6e2 +size 158534613 diff --git a/checkpoints/model_weights_000002621440.pt b/checkpoints/model_weights_000002621440.pt index 56a2f7f1419007aeb2dbb531eaf40d7fed93600a..bbbfbae51f49a12e51a2f2c2620e4f314732cbbe 100644 --- a/checkpoints/model_weights_000002621440.pt +++ b/checkpoints/model_weights_000002621440.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f728fc79cda08757ad883858724005cb0e61d52e0efa16249a5a4e1495453b8b -size 234226143 +oid sha256:2e0b343aee76bde8a19dd897a42ce14f0451252ab9b9fd4f335065a92aa8605c +size 158534613 diff --git a/checkpoints/model_weights_000002883584.pt b/checkpoints/model_weights_000002883584.pt index 89094b78fd732036110269334269ef31207e88c0..78b46004f52a74670562e16bd30b5b49d5626ae5 100644 --- a/checkpoints/model_weights_000002883584.pt +++ b/checkpoints/model_weights_000002883584.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cb1f20d25f77bdf93c0307f2ca0aee737dfbccacadb09f7e7e364d7635e46ebe -size 234226143 +oid sha256:f48f4684606ef481f2c80a61a4e3ea837966aa6c83b00a921962cb2dcfc0142c +size 158534613 diff --git a/checkpoints/model_weights_000003178496.pt b/checkpoints/model_weights_000003178496.pt index 510b0f7a8d19e05e5b3bea697d098d9e161fd40d..d8c416ae1c12aa728985e96b56843e445f9777ca 100644 --- a/checkpoints/model_weights_000003178496.pt +++ b/checkpoints/model_weights_000003178496.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:424e084a912b5d8d2760b82daa80a209fa37dc42e2c77bae49d5d71c8d993a91 -size 234226143 +oid sha256:d3c2b7994ba90741861a9edce08369b7beb3cce479a00dbad00321dae6ad4d46 +size 158534613 diff --git a/checkpoints/model_weights_000003473408.pt b/checkpoints/model_weights_000003473408.pt index 539ec0de6d2d2163693936929940d982b0b8f82f..d6cdc7610c90640d71bbf00583bccc0d689ab2a1 100644 --- a/checkpoints/model_weights_000003473408.pt +++ b/checkpoints/model_weights_000003473408.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a12e7df724b5697fff28f280036e9a8b11b9428274e089adab7779ec77e8f574 -size 234226143 +oid sha256:76f7461a1992a3572be157db3b0cab7013d80085a94a5a8d19fe853425d809de +size 158534613 diff --git a/checkpoints/model_weights_000003833856.pt b/checkpoints/model_weights_000003833856.pt index 80422291ac54fc201c102ec8b788722ba80136ef..be0e8001be356924e0d1f61f1549cb5b2c830776 100644 --- a/checkpoints/model_weights_000003833856.pt +++ b/checkpoints/model_weights_000003833856.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:66e1fb23f55d74be5473a2117c33f539f67fa884f9ece2d90ca7e29551868242 -size 234226143 +oid sha256:aedeaa01019b66ca79ba7b43014f45fb50a812f730f16febe1d6a3b48f62615d +size 158534613 diff --git a/checkpoints/model_weights_000004227072.pt b/checkpoints/model_weights_000004227072.pt index ec0043b4860e679e61f4e88b86debc2f4e457237..c12134977d2c616bee3d5a12ab1d3aaa05b7a71b 100644 --- a/checkpoints/model_weights_000004227072.pt +++ b/checkpoints/model_weights_000004227072.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:77b7b9f348b63e11fdbd62ebe813651e3ebb38cec81a52811c3ba752bc1bbb64 -size 234226143 +oid sha256:f0525e01ca26d91956694820502f61198ac78a1ccc3b298db3d95d665ceb23ac +size 158534613 diff --git a/checkpoints/model_weights_000004653056.pt b/checkpoints/model_weights_000004653056.pt index 0ee9ba4fa2d99b5fb0040c67f617ed798ac10c94..cab18a4dd9e1b144a8ff5566c4fe8cf41f4c03b0 100644 --- a/checkpoints/model_weights_000004653056.pt +++ b/checkpoints/model_weights_000004653056.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:247721eef02d0a40d746e0c970619ab1724baa3e47618f87825d19a68a48961f -size 234226143 +oid sha256:3e491ed28a3b874e0dc41eb160acde5f00b55065df451abf8b6570734555b4e8 +size 158534613 diff --git a/checkpoints/model_weights_000005111808.pt b/checkpoints/model_weights_000005111808.pt index a05d4d1b83077517aa0cd3ac461f85559730b0db..93e882f968ddb826fc94ddb060a7f43ac4c7d372 100644 --- a/checkpoints/model_weights_000005111808.pt +++ b/checkpoints/model_weights_000005111808.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6f6608a08aa882a6e995b9ea23f36252831e699cf52a27de6ef99361212222be -size 234226143 +oid sha256:ccf3d39776532b36b19f3ec79b60b5fd434f0b16e4fa8c072f3b3d57df7826a4 +size 158534613 diff --git a/checkpoints/model_weights_000005603328.pt b/checkpoints/model_weights_000005603328.pt index 8abfd7cfeef613562de9a8b0429a769d1b50c9ec..5f6b522ed172054e38fc78bef33f53e828d6605b 100644 --- a/checkpoints/model_weights_000005603328.pt +++ b/checkpoints/model_weights_000005603328.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:17e22e7c1a2d5361b4f46bdec1ba8d89aecfd0ea61840de7f41f144338c24ff6 -size 234226143 +oid sha256:f5c2e68de5e4b0d32c67eca92f9a0937da69b8e74f85cc409516716cf789db3a +size 158534613 diff --git a/checkpoints/model_weights_000006193152.pt b/checkpoints/model_weights_000006193152.pt index 3590396df7926ac7d04a83e5f571082d93a132ac..af0b8b556740ce542fde419361b7c2c86f46ff62 100644 --- a/checkpoints/model_weights_000006193152.pt +++ b/checkpoints/model_weights_000006193152.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f92c1d9982c936b2b0fb9c14d3a06b86e54375bf29af2306279d5644e2585816 -size 234226143 +oid sha256:b6ef217b4aad17fc10dbc49d4dd1ba6d42c06389d30e493045e85e287a3bebd2 +size 158534613 diff --git a/checkpoints/model_weights_000006782976.pt b/checkpoints/model_weights_000006782976.pt index 51c7baf8894c354b5847cf1735dc1d642e65009f..9d08475956871acd1098b55dbbc710d3aaef5d33 100644 --- a/checkpoints/model_weights_000006782976.pt +++ b/checkpoints/model_weights_000006782976.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:60f21c9fb16f352e55ae929c3d81eb0c0655cccb602b02a28f496870e4556e19 -size 234226143 +oid sha256:3944d9999d0a3394a13252e0d7830ff1f2e346b4e7c98c7edddeac90e1df7db8 +size 158534613 diff --git a/checkpoints/model_weights_000007471104.pt b/checkpoints/model_weights_000007471104.pt index cabc5878d652bee1fbf7c0ca81f6128088126bcb..319cfc3d91797bf0c6946f264ffeeb5e4c08462e 100644 --- a/checkpoints/model_weights_000007471104.pt +++ b/checkpoints/model_weights_000007471104.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:34e6101f8430ba83678c0fe59f814137957e3095fcfc8517fc966423ed666b7d -size 234226143 +oid sha256:54bb80cd889e0ebb0c56cc727d807ec4f66d700e9b4c383020303d36dd269970 +size 158534613 diff --git a/checkpoints/model_weights_000008224768.pt b/checkpoints/model_weights_000008224768.pt index 55c57f20068a9959f96b6a7181b55493f7ba343e..7c49f66efdda4a6e7877a7eaae61ac572996f966 100644 --- a/checkpoints/model_weights_000008224768.pt +++ b/checkpoints/model_weights_000008224768.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4298ea42d8fa0210ebd70c8bb5e7e95b522d938528d78c3c4b09991c2ba99814 -size 234226143 +oid sha256:4bcb23a654815dd17e692e884013d1543bbb8ffdbe88ef5fb4ac1282b9f02f8f +size 158534613 diff --git a/checkpoints/model_weights_000009043968.pt b/checkpoints/model_weights_000009043968.pt index b2507f9b3bce55be93ea093ff0f5283833094ac5..e59b54d208f5b6a8cea26ff7f84c4829e345d48b 100644 --- a/checkpoints/model_weights_000009043968.pt +++ b/checkpoints/model_weights_000009043968.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:854e476019163e43b52d792f911f844c9198cb2373599b65b12092cc12a9d596 -size 234226143 +oid sha256:ca8a0e81c0bfcde3d5d4e691321562bdc6c5292ceea21019d331f40154011ebe +size 158534613 diff --git a/checkpoints/model_weights_000009961472.pt b/checkpoints/model_weights_000009961472.pt index 6417ec5e289c85e6ad773524946a0d1fbb15fedd..caf12223adc6e79de8ac21d30197beb0f96b5775 100644 --- a/checkpoints/model_weights_000009961472.pt +++ b/checkpoints/model_weights_000009961472.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:01cca75622fc160b12682ff7fb7ec3486274a98dd0b49605fcde9e669e080797 -size 234226143 +oid sha256:2f991720cfc930ed4c17fe98e8a64a67699218fc9e8d58239f181f1010a1c225 +size 158534613 diff --git a/checkpoints/model_weights_000010944512.pt b/checkpoints/model_weights_000010944512.pt index 076c4917ab259f0f24c9bc87da48866c135e835a..03b3fd252848270d29468ec7eeb4464166c28cd8 100644 --- a/checkpoints/model_weights_000010944512.pt +++ b/checkpoints/model_weights_000010944512.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2c51399248c5ceb6ee13416c1e6c35c4271cc52e83ef931217399e6635bf777d -size 234226143 +oid sha256:05d549ba40c477fd47bc1076c3af309976c8bd64e9bd2b3c1305ceac2228e992 +size 158534613 diff --git a/checkpoints/model_weights_000012058624.pt b/checkpoints/model_weights_000012058624.pt index 2955fd04fd245ac7e57296c48fe222f45181a7c6..43916a370629412fb53277622e53f1221654378b 100644 --- a/checkpoints/model_weights_000012058624.pt +++ b/checkpoints/model_weights_000012058624.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f3a74f634633a278a58209572058ae827ec7e01ee5a96696045bb3f93afab04b -size 234226143 +oid sha256:2958b18f765d72d8140e657633d0306fa3621a126ceddce1721235afd5b1b076 +size 158534613 diff --git a/checkpoints/model_weights_000013271040.pt b/checkpoints/model_weights_000013271040.pt index d9941d4d968286fdbae786d179b8a6cd9965360f..20dd48b38341b15cf31f14a207459fccc6db6d0a 100644 --- a/checkpoints/model_weights_000013271040.pt +++ b/checkpoints/model_weights_000013271040.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:53155fb2890011ad150acc43f6003581a314c9bd6959a97ad543617bb1d53647 -size 234226143 +oid sha256:bb3aad43dff48c78ebbd28386163ce096b002230b6270e3df1ee692d93484531 +size 158534613 diff --git a/checkpoints/model_weights_000014581760.pt b/checkpoints/model_weights_000014581760.pt index 364af4a9a7c7f3eefcce4575da2505caa22ad969..da2b3e7a21d5d5f871ed739fe2ad55ef160a82a4 100644 --- a/checkpoints/model_weights_000014581760.pt +++ b/checkpoints/model_weights_000014581760.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:98682e9f320585ccba1290bf5ef7b2137c49a1798377da25b44f50d6ade4629c -size 234226143 +oid sha256:081fe651fd6a624635f504b535f45efc57f6a3fb4d8113fb56cf375c3e3a25c6 +size 158534613 diff --git a/checkpoints/model_weights_000016056320.pt b/checkpoints/model_weights_000016056320.pt index 0514bdf48148ba8737ceaa9bafd763f0a3a6e426..70bcd02c17de8b3ea89d07730f69c34a92502462 100644 --- a/checkpoints/model_weights_000016056320.pt +++ b/checkpoints/model_weights_000016056320.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:83681036defad3242b3cc385e4fbffe11670693e2fdf6fe99a08ccc322eb509b -size 234226143 +oid sha256:1b952c9c9400f67d2709e483164eabddd2724eaaad19db09910b87a3f14d34bb +size 158534613 diff --git a/checkpoints/model_weights_000016384000.pt b/checkpoints/model_weights_000016384000.pt index 88c5b13117f9b7f9f31fb0a4ac3c62f2cfc6e57f..a7c8420a6dff7b8cd0027150d0eb105095b99b20 100644 --- a/checkpoints/model_weights_000016384000.pt +++ b/checkpoints/model_weights_000016384000.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a971fa39b77f0e1501d61343e528b5986733d09f30cf9b5ee9b7570588e617a7 -size 284687203 +oid sha256:bc9b556426b1799885ac629a47a7f33ed6103b2d7cb428d4106e40243dc4b296 +size 158534613 diff --git a/checkpoints/model_weights_000017661952.pt b/checkpoints/model_weights_000017661952.pt index 06b95902d9fa525f18cd330886f06a47499a3388..0b359a4e2984b1d9f032fe3514dbd5b18b990ee9 100644 --- a/checkpoints/model_weights_000017661952.pt +++ b/checkpoints/model_weights_000017661952.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f7ba7aa2beb95e2579537b631a8f6ab19053cb730a04a501395f8d3ad5a845aa -size 234226143 +oid sha256:ef75e962b044d6b1090145193db24cc110cc9f9d669f47079819048c3c03ba5d +size 158534613 diff --git a/checkpoints/model_weights_000019431424.pt b/checkpoints/model_weights_000019431424.pt index 1fe831a15d8792add2fafee2459818a5afd0dec9..05c6312e45ed10de5778203509da82000152a22e 100644 --- a/checkpoints/model_weights_000019431424.pt +++ b/checkpoints/model_weights_000019431424.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e0c35c25503b15b80089e09854214bb7782ff7366474833fe72e54143e4f172d -size 234226143 +oid sha256:279173f0048126c4595983cdeba13be1477866ba32992552246ce047297e4fd7 +size 158534613 diff --git a/checkpoints/model_weights_000021364736.pt b/checkpoints/model_weights_000021364736.pt index 2554d52655e083f97470ada9644fa8677fc56dff..aaf6bd1e9da1e1a25188c88e91f372c6057420e0 100644 --- a/checkpoints/model_weights_000021364736.pt +++ b/checkpoints/model_weights_000021364736.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:08f53fc8f71b546c0c5be5fa3ef779a7e36d812fe78db08927da8b08c96c8977 -size 234226143 +oid sha256:c596877ba636cf0120e442faf544d3bb9426bc8ecabd1d561e370b8df7614826 +size 158534613 diff --git a/checkpoints/model_weights_000023494656.pt b/checkpoints/model_weights_000023494656.pt index 4c493a78da3df2b832f4c763c16af9649576ddf7..aeb5624c0aef3c494c76f982a4c6b44fa32dea8d 100644 --- a/checkpoints/model_weights_000023494656.pt +++ b/checkpoints/model_weights_000023494656.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cd1eb2c66b9bb9020998baf0a974fc68c92ae6bfa98124e172bf8dd3c6b6a411 -size 234226143 +oid sha256:44d3769fea28b9abc33af31cc0c9ef324bfbff2613b87ef33ef1073a5383d9ed +size 158534613 diff --git a/checkpoints/model_weights_000025853952.pt b/checkpoints/model_weights_000025853952.pt index 3a7e8b204ef3f537974dbaa4c561b329aaa47fbb..9ef278a625966789571865e829331a8f5bd9197d 100644 --- a/checkpoints/model_weights_000025853952.pt +++ b/checkpoints/model_weights_000025853952.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:488368a572ee91b15a723f0f866444e638dc460d0e18ceffc6b4cbf6a4e81c06 -size 234226143 +oid sha256:63849c3262775782706ffe6f05ad7bf373992b8f16351f55efcfb7864fe87cc1 +size 158534613 diff --git a/checkpoints/model_weights_000028442624.pt b/checkpoints/model_weights_000028442624.pt index 3867e39044f12c1fcc6bc0aca23a25d1ca582ac8..f8e4482cf88357cf2bbc6fa43dff4a5aab1f2730 100644 --- a/checkpoints/model_weights_000028442624.pt +++ b/checkpoints/model_weights_000028442624.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8a930faffe752cb358c54d987904d347ab7694a35a95f0c5b07c3593686fd436 -size 234226143 +oid sha256:02e0f22555d7830ad168a9b5b147275403bfcba32eebe0ab97fa99fc9bb7246b +size 158534613 diff --git a/checkpoints/model_weights_000031293440.pt b/checkpoints/model_weights_000031293440.pt index 316a024dd5f83352bc2118900a323962b76ab790..6da2b14d3dc042d63ecc3bd8393ad5dcf53790bf 100644 --- a/checkpoints/model_weights_000031293440.pt +++ b/checkpoints/model_weights_000031293440.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:dd17484832cfb8608f78df72c8ac2d11ca9596bf3ff436b6eb2e4e1cad23f5ee -size 234226143 +oid sha256:f6261b73671c4ee28cede538638e5f75e2e65b1defe334b50c805755977e2d2c +size 158534613 diff --git a/checkpoints/model_weights_000032768000.pt b/checkpoints/model_weights_000032768000.pt index 311606a121e0e6fee14d373b1e40ea8487926035..df566dfdb55d847392004be38fb9fb1a6ebf3cd4 100644 --- a/checkpoints/model_weights_000032768000.pt +++ b/checkpoints/model_weights_000032768000.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6e37512e6a3522edc964c7ad6e99b681ae41caf8fb0266db04a0cc7a50157456 -size 284687203 +oid sha256:681fbecb33f42a2e3c1d1e4fec017c084565900784aaf577a4cc91c6ef4ac7ed +size 158534613 diff --git a/checkpoints/model_weights_000034439168.pt b/checkpoints/model_weights_000034439168.pt index 71820fb741c4709f03503d12fa470dc26355cd1c..2018c78c0a419e78d1991da5ed8c4cc0a501b62b 100644 --- a/checkpoints/model_weights_000034439168.pt +++ b/checkpoints/model_weights_000034439168.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9b8e201de0aa04157fe850fcbab9bb6300f0a71114b9240aa5387c958c4e4453 -size 234226143 +oid sha256:f90a813a7d709ecb87dffb0a566886526e1cd19f5e604f13d133389f2a70205d +size 158534613 diff --git a/checkpoints/model_weights_000037879808.pt b/checkpoints/model_weights_000037879808.pt index 1966638e70807baed85dacf03e5cfa25f9d1456c..7666ff947675f9daba3b1b024490824fdd4e0db0 100644 --- a/checkpoints/model_weights_000037879808.pt +++ b/checkpoints/model_weights_000037879808.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8795d7d0636032dea7d7ef81e5111a88940dad649ebad0d91c82a4c28c713adc -size 234226143 +oid sha256:39a04739e91071ba542a0bec45be466945066ec460d2e3a1425d23d1028ee41f +size 158534613 diff --git a/checkpoints/model_weights_000041648128.pt b/checkpoints/model_weights_000041648128.pt index 6a4888864cc59abfaed17a7ba83d595d888ef3ee..8477f6a23e7df8d0b7d42bf2123c019fd4d847d3 100644 --- a/checkpoints/model_weights_000041648128.pt +++ b/checkpoints/model_weights_000041648128.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5a8f39e4306d5aa3b47e64148d033e545b834ef35cfa46cf657fe22b058fdef1 -size 234226143 +oid sha256:1d594de434bf3ddf19e1bccdbcf367e185b3e0ace4912d896fbcb15eb13e753b +size 158534613 diff --git a/checkpoints/model_weights_000045842432.pt b/checkpoints/model_weights_000045842432.pt index aaa08f48a4da7ad3929a773ae1d6214701e92f2e..6751b6d4f9b0ba3c716a37b7ead2a6932bc17875 100644 --- a/checkpoints/model_weights_000045842432.pt +++ b/checkpoints/model_weights_000045842432.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fe91d7a212fcce8c1dc14df5844fe6c3d2b89633349ab7ea0bcbaf8c0f7def29 -size 234226143 +oid sha256:19c96bdff45c8fe439e9b139fbbca31c0f259c093d87c8bbf8b39c516b7a6a31 +size 158534613 diff --git a/checkpoints/model_weights_000049152000.pt b/checkpoints/model_weights_000049152000.pt index d0be8a2077f9d73023097d479c58944aab678e13..b6171e8024ebec3bf3afa05603faf6db1c42a164 100644 --- a/checkpoints/model_weights_000049152000.pt +++ b/checkpoints/model_weights_000049152000.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:184d0f82bc99a936ab4bbb8440cc8270c12f5196b72c3be323f74a17b73ec5ab -size 284687203 +oid sha256:5338c5a8c553f4436c3ca6ed5a8c9cff7f5f8b74103a9c9261f851a79d8ef1c0 +size 158534613 diff --git a/checkpoints/model_weights_000050397184.pt b/checkpoints/model_weights_000050397184.pt index e61d62108fb4ea7aa58967b218f92045cb5b7595..c5eb1c0700032dfdc37dd19397b93d1065c52271 100644 --- a/checkpoints/model_weights_000050397184.pt +++ b/checkpoints/model_weights_000050397184.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:06db438684ac92e74939d7ecc86aad8558416a405a2ddf361b316e0d768a6b32 -size 234226143 +oid sha256:f04f4470d81fa43bd56e795ded6b1444c99803a625f014a671c457da3eb63ac8 +size 158534613 diff --git a/checkpoints/model_weights_000055443456.pt b/checkpoints/model_weights_000055443456.pt index a4d09cb08658ad2d42461112d7573dd7917222bc..ab4ea151e7b601cea100c91de1dc4c2399956ca7 100644 --- a/checkpoints/model_weights_000055443456.pt +++ b/checkpoints/model_weights_000055443456.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b451b39c0ea8ce9c3a6bec718d46aaf8f7a09cf72cd8169955855ef37379fe8f -size 234226143 +oid sha256:dc198671d67ce6a5da1c3dfa0b1b3323bc916d185ad513b8fe3fcc09f43aa35e +size 158534613 diff --git a/checkpoints/model_weights_000061014016.pt b/checkpoints/model_weights_000061014016.pt index d13e5182f3d30ca82e5ff81fd311d5480487a0df..a9c75fe0f707edca7a2ca5108a5ecfe681b88175 100644 --- a/checkpoints/model_weights_000061014016.pt +++ b/checkpoints/model_weights_000061014016.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3e0a928edc7cd404af61c89de19eed98e127d97b416b0c47e837acecd6286d10 -size 234226143 +oid sha256:9e6d1e5d15aacacbeba49b8eafab3a8cc08d7cb8b775f0df88bc9af8b3e98f02 +size 158534613 diff --git a/checkpoints/model_weights_000065536000.pt b/checkpoints/model_weights_000065536000.pt index cd897e5707ea606db332329a6d77b268f6d9cb53..64d14db0a09d9e45f935a8091d3eed8e03ad6cba 100644 --- a/checkpoints/model_weights_000065536000.pt +++ b/checkpoints/model_weights_000065536000.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6bf57cd0c89b53c0ab805f25e16bac1ded3d6aee1f901b52a07969a79ac70cd6 -size 284687203 +oid sha256:69983950515d573336f3cb81d19dbd13c1f79728c7c216f89515bea6d80e23ee +size 158534613 diff --git a/checkpoints/model_weights_000067108864.pt b/checkpoints/model_weights_000067108864.pt index 2d59299fe2afcef2b9bd28c3c077d67906b2541b..7a1b9e27f92d469eba2be5ce7e1ccf7f3ff99b8c 100644 --- a/checkpoints/model_weights_000067108864.pt +++ b/checkpoints/model_weights_000067108864.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:46bcafd243add89b20b23d4097ddc7efdea9e117cfb025edffaaa18550e478f3 -size 234226143 +oid sha256:edcf04a9e05aa8d24c1acef9956763a6fdfac8226fd49783fbb0c12f13910757 +size 158534613 diff --git a/checkpoints/model_weights_000073826304.pt b/checkpoints/model_weights_000073826304.pt index e53e991516a3389b7603dd4583f5f93d5278fa45..fab0377f89d1fb840b50be6831f0f0a52d0f2f8c 100644 --- a/checkpoints/model_weights_000073826304.pt +++ b/checkpoints/model_weights_000073826304.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b3de99fa60c89a43bcda5d3d04dcb628de952b530161db6d1619cb01d26fee98 -size 234226143 +oid sha256:3b97cec5692994f536f0fb54e9fb574cbc81828402ae77ef3a278e5cdea18ada +size 158534613 diff --git a/checkpoints/model_weights_000081199104.pt b/checkpoints/model_weights_000081199104.pt index 590d5df557a6110a963bd0f83b7c4c4fae5ed696..0ab61ab92cf4bfdbd622f78f5c3bcde6228231ec 100644 --- a/checkpoints/model_weights_000081199104.pt +++ b/checkpoints/model_weights_000081199104.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c85e8965bbd8347a7155dd82b37c802daa744efb84745e1782a84081bdb773e7 -size 234226143 +oid sha256:b03ec4cdaf3f8288d5b7cf2a950e24cfe6de712f58bf42486ba11d9b9ea8b63f +size 158534613 diff --git a/checkpoints/model_weights_000081920000.pt b/checkpoints/model_weights_000081920000.pt index aa6c1b4b940fe2108808ec99ba34afd122887953..3cf259075c32383d7ea6cbfb063f97a330ec21a1 100644 --- a/checkpoints/model_weights_000081920000.pt +++ b/checkpoints/model_weights_000081920000.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1e5cc965285861e6cc6348e58d5efd5e7dafbbbc0e15fb388e7567f72c7e9c7d -size 284687203 +oid sha256:307af649a87bf803077e543c693f061deda200d42f42f4a08c2a3b5ba81502e0 +size 158534613 diff --git a/checkpoints/model_weights_000089325568.pt b/checkpoints/model_weights_000089325568.pt index 1b56291ea59961c8bc9c04aa6673b821224cbbde..43071eed39a099cc91b8dec256c5817247f22daf 100644 --- a/checkpoints/model_weights_000089325568.pt +++ b/checkpoints/model_weights_000089325568.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9e1db1bbb93b348314618ed80819c031deb8b450dfe4e1f2acc7fbf9e8b779a5 -size 234226143 +oid sha256:09bfc47afa637c968ed8420e5c7ccff45ce08360fe888ea91bef9764d22fa468 +size 158534613 diff --git a/checkpoints/model_weights_000098271232.pt b/checkpoints/model_weights_000098271232.pt index f38d739e2eae6a1603b014c3e78d93ece4c46524..79991325f5a6bf6a6cf8c23a2e70616b33f09ba3 100644 --- a/checkpoints/model_weights_000098271232.pt +++ b/checkpoints/model_weights_000098271232.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:338e71fc44b43448b2d3f3b034e15bd612cb868e35658e83822a3d4e972a3437 -size 234226143 +oid sha256:08cffb75348e3955cd70a0648b202e3d82c0bcfbbf8e48cc4d4fe2b619f07b9f +size 158534613 diff --git a/checkpoints/model_weights_000098304000.pt b/checkpoints/model_weights_000098304000.pt index 1f9d61bfc28e8b7dd5eb38378ce97ebdec2e3ea9..3e885dae8777fe7eaa21494f65acc8d8b6e957b9 100644 --- a/checkpoints/model_weights_000098304000.pt +++ b/checkpoints/model_weights_000098304000.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4d39b3954672ba6327acb792d9645e958b95085d734188a8badc42fbd2b08c08 -size 284687203 +oid sha256:7108ff3800ed49a068dc33517a8145f06f04201f2b4c27550e4bfaa9685856bc +size 158534613 diff --git a/checkpoints/model_weights_000108068864.pt b/checkpoints/model_weights_000108068864.pt index bc70b3434a43b2791e21280701dabba2e2263391..ee6fbf85867f3ef2262735c83d2b12ee4f4b238f 100644 --- a/checkpoints/model_weights_000108068864.pt +++ b/checkpoints/model_weights_000108068864.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d9ad7b6e22daf4d41497117ce2cac4f3db44182c86c442f53155c80219bbb0c1 -size 234226143 +oid sha256:4b490d1540e7621213e56b991973c101cbf4aa1def19a86491095cdd5971a374 +size 158534613 diff --git a/checkpoints/model_weights_000114688000.pt b/checkpoints/model_weights_000114688000.pt index a69f1d4cffe2c8430140e574504210986798c7da..0d4dbddf814cd80c826c342358ac073a99bdc101 100644 --- a/checkpoints/model_weights_000114688000.pt +++ b/checkpoints/model_weights_000114688000.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:24642db8b874beafee44da75ad14c9218d25a158c2e9033c5a28c3007c04600a -size 284687203 +oid sha256:9588d016cf12cb349e06c40c9df607aa237017d3e78e33de26dcd309747d368e +size 158534613 diff --git a/checkpoints/model_weights_000118882304.pt b/checkpoints/model_weights_000118882304.pt index a89c1570145ecdd9b5707e2b76a4f50b6f9ebee0..9e7132ecc40160d4cf22e07d6e49557e23c720ff 100644 --- a/checkpoints/model_weights_000118882304.pt +++ b/checkpoints/model_weights_000118882304.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:34b950bf9a41abcfaceae04695a1912b5be2e71b662816a605cbb19008f71e8c -size 234226143 +oid sha256:ab95e8d614027d558734ec83941fa560e0e9d9a7e4e64505ad2e6b22f070e7a4 +size 158534613 diff --git a/checkpoints/model_weights_000130777088.pt b/checkpoints/model_weights_000130777088.pt index c82593bc1c8bf935fcda384fdf5c0da1f3652208..cabbb8d8a7e4b53db9a25c839e666d08c43db8e0 100644 --- a/checkpoints/model_weights_000130777088.pt +++ b/checkpoints/model_weights_000130777088.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3f52da6c35e1be7d6b6a92686d0111af451f3fdc0a36f6f7f32adb21e344d8c1 -size 234226143 +oid sha256:e3038d99e2842e6be386ecd124f3d413f892bf84fd6557494b5dab12a814d897 +size 158534613 diff --git a/checkpoints/model_weights_000131072000.pt b/checkpoints/model_weights_000131072000.pt index 9dd3c99e65863e109bfd467baa406b80a3ecb2a4..a6003abc4dbd1d61429f715f508cd8116b560a8b 100644 --- a/checkpoints/model_weights_000131072000.pt +++ b/checkpoints/model_weights_000131072000.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5ec9bb020e86634dccc22e9ad92e7d9f4242a638bc38ab0db0383fa8ac92877b -size 284687203 +oid sha256:4ead5e395624c577562694f4c0cd680c7dbe55d4baff4f7c430d2ce2ac1887e0 +size 158534613 diff --git a/checkpoints/model_weights_000143851520.pt b/checkpoints/model_weights_000143851520.pt index aabd3d12c36f1745ff17cf4a6d7266f9e58171b4..b99e76f0a9d76d1545cfaebb13e94991843b2947 100644 --- a/checkpoints/model_weights_000143851520.pt +++ b/checkpoints/model_weights_000143851520.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cd4e9687b4c43d505f4687f04ffc0b051b6020890afb17dc7529c181d7b0c368 -size 234226143 +oid sha256:33cc863a4959806a5fb9b18bbbe21ced4544196c4a84321e0bc59cbac5c5e7b5 +size 158534613 diff --git a/checkpoints/model_weights_000147456000.pt b/checkpoints/model_weights_000147456000.pt index 1ca29278e622fcec1c95ad33667abf73c5ac345e..ae60fddb6fa0b8ced5957ce9d96b600e4b7b8375 100644 --- a/checkpoints/model_weights_000147456000.pt +++ b/checkpoints/model_weights_000147456000.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:196925b562fb0e8d2beeec1387f4aa161724617b1b7167125b640b593487f509 -size 284687203 +oid sha256:5703695ac9e71f5d3b8bb34a1dd9f84b78ca2997d3537f28f0d8f109e43feaa6 +size 158534613 diff --git a/checkpoints/model_weights_000158269440.pt b/checkpoints/model_weights_000158269440.pt index 2d735f4cb9451eb33b82aea4378ce0b214dd7665..2d1c994c8e165eb356cdd35d42fb2a646ec2c0f8 100644 --- a/checkpoints/model_weights_000158269440.pt +++ b/checkpoints/model_weights_000158269440.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c65078a8240e0621ba3b12596c8c3f680f0c733841722d11641e20271bb6a845 -size 234226143 +oid sha256:e0c57f58d1b5c979a21fc81f6c192074b94a6459dc247a00ffe6f6706105db15 +size 158534613 diff --git a/checkpoints/model_weights_000163840000.pt b/checkpoints/model_weights_000163840000.pt index 737ff50d297c31b87a808c3178513512e1fb02ff..33fa20d981edf8fe8973e44b4dd86cbe0907e7d7 100644 --- a/checkpoints/model_weights_000163840000.pt +++ b/checkpoints/model_weights_000163840000.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:34fd0a241eb11771c76231347a47e1d51993ed610368b505ea7a9a7debbaace1 -size 284687203 +oid sha256:371b8b2995edb994e8db8fabb306ad01869f8e8699983678f2c916a371318873 +size 158534613 diff --git a/checkpoints/model_weights_000174096384.pt b/checkpoints/model_weights_000174096384.pt index 0145419d784d5739745fb198e5c27a389b289df2..44fc8abb836c3b87c4b5eb96a57b9a9927052cf7 100644 --- a/checkpoints/model_weights_000174096384.pt +++ b/checkpoints/model_weights_000174096384.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0de60c4154f8f65b2911c5e5578f57f1e5042d7e9a570c823d2e54f77e46425c -size 234226143 +oid sha256:5bb27ef536c16340d5f784b536d98ab9308299ddceb7184b96eef95f136229f5 +size 158534613 diff --git a/checkpoints/model_weights_000180224000.pt b/checkpoints/model_weights_000180224000.pt index 8f300a2d5379a38c23caf64779c11463f57ca016..4e0bc565262443b6d8b9096980fc6a24a2070efb 100644 --- a/checkpoints/model_weights_000180224000.pt +++ b/checkpoints/model_weights_000180224000.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b20f535cb26260ca3cbb0a3a06005f60b11caf81cf51519ca91f10b5a9015dd0 -size 284687203 +oid sha256:7a3d02f19784fb6f99a2c28703505c21bc3c30bb46b2e515e0450b18919a5db7 +size 158534613 diff --git a/checkpoints/model_weights_000191496192.pt b/checkpoints/model_weights_000191496192.pt index 16555d2d1eb388fe7b327d4b74943889c8b7d832..fa7b5c8f8a5682691d97d4133814f82f5e8ea3a0 100644 --- a/checkpoints/model_weights_000191496192.pt +++ b/checkpoints/model_weights_000191496192.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:934ba6c4eaf145cdef1e42b42f134e5cd79cd6d5ba30f96700a1bd4635e21f79 -size 234226143 +oid sha256:4557503edb4ae73695dc9800474fc0f1e35aa44f1772ca908e1e04dc6966f96b +size 158534613 diff --git a/checkpoints/model_weights_000196608000.pt b/checkpoints/model_weights_000196608000.pt index 2baefcb60540b66d737fa88ef932314a9bcdc88c..34e185b83a30fb605a6f6ef0eecd4505cc94f7be 100644 --- a/checkpoints/model_weights_000196608000.pt +++ b/checkpoints/model_weights_000196608000.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e3a315c7f76678e7b4980f41f268af267201657fb9787514f776e8b84e68c7d0 -size 284687203 +oid sha256:08fcd5ee3e9fc87edff3178d3a17283e340dcaa07226ebc192cd888cbf2edf3a +size 158534613 diff --git a/checkpoints/model_weights_000196706304.pt b/checkpoints/model_weights_000196706304.pt index 87413e3b81a437dc6a81456324543df0f7efdead..705b2088f9271b5636a6159d53fe252679ca6f66 100644 --- a/checkpoints/model_weights_000196706304.pt +++ b/checkpoints/model_weights_000196706304.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ef39db7ad7139b8a7f4bab77435ce6bd4e6579515b8a135949d2f2b346a13e8d -size 234226143 +oid sha256:74d99e12a6c1ed110910596b8d5a037089fd2611dd83556da79330b3656012c5 +size 158534613 diff --git a/checkpoints/model_weights_000197361664.pt b/checkpoints/model_weights_000197361664.pt index 58ba66b3f26734457ab9748a0016e433372a3e54..90a006f064157ca51982bd525f619d26ae4aa358 100644 --- a/checkpoints/model_weights_000197361664.pt +++ b/checkpoints/model_weights_000197361664.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a839970ad54d993bb31d3bde0311c0a1dbdffefcd8f2285eb48b5622b509779b -size 234226143 +oid sha256:1b7739f84c9f75efaf936234b77c3c42f8157fb497d20f36e15e4464a45b7008 +size 158534613 diff --git a/checkpoints/model_weights_000198017024.pt b/checkpoints/model_weights_000198017024.pt index 59a81bf675d6d92f3707246c64b5788b88886f62..afe4013356f399b424be19feb9973fcb10d0772a 100644 --- a/checkpoints/model_weights_000198017024.pt +++ b/checkpoints/model_weights_000198017024.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fb31b63f40c200bb434543831190875d1fb736d94c82872ab676bb5b2fee9ba8 -size 234226143 +oid sha256:d8fbbe2652335d842110eac2604e29f18a5ae5f63dd17c67b4a676778331a097 +size 158534613 diff --git a/checkpoints/model_weights_000198672384.pt b/checkpoints/model_weights_000198672384.pt index d8462a5cab81304d3c2fe9e85730b284c4ad4f06..028e80306fd764558d6050e9025970f737f7cab7 100644 --- a/checkpoints/model_weights_000198672384.pt +++ b/checkpoints/model_weights_000198672384.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:12e823b683e61e7cde11b92c1ffe1320d97ecfb0c8bbcfd3cbf8f2a3d0189e92 -size 234226143 +oid sha256:27421b73e5184b050643d51431b10ab04a2158fb44fdd69a0cf68c531ea7d14e +size 158534613 diff --git a/checkpoints/model_weights_000199327744.pt b/checkpoints/model_weights_000199327744.pt index c37b17dd766a426d4b2d2bd4fa83e4ef36166d84..110b257d6917da63d16a965c0d4370ae9529b474 100644 --- a/checkpoints/model_weights_000199327744.pt +++ b/checkpoints/model_weights_000199327744.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d81f5635d528a252848e3941c25c68eb0bea66c831e541ddf480f54f876a5e3d -size 234226143 +oid sha256:b67b10b24c8390a8bafbc84bde060a0378b4db56bcfb46e0eb0b7659468c2b81 +size 158534613 diff --git a/checkpoints/model_weights_000199950336.pt b/checkpoints/model_weights_000199950336.pt index 9f637fdc68dd587bfa714acfb915cc39185f04d9..4281d18950f23242505da3acf12fc7fa5ff17bbb 100644 --- a/checkpoints/model_weights_000199950336.pt +++ b/checkpoints/model_weights_000199950336.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:191a3ab543909a00d9a77cf97897f402074243a69b2e8a05a74dcccf0d90195e -size 234226143 +oid sha256:48e78db684661eec2c311e6d98aacbf1943f769a0e8ba9d7f9ef0d19a13631b0 +size 158534613 diff --git a/config.toml b/config.toml index c5f97c04febc17e726433a1110e02fe96e423bd6..28afce61c94e1146ae3f0014e484407fd3cb1f17 100644 --- a/config.toml +++ b/config.toml @@ -1,6 +1,6 @@ -model_name = "pile_llama_replace_17367_L8_1024" -dataset_name = "eoinf/PL_Replace17367_L2_alldataset" -n_layers = 12 +model_name = "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2" +dataset_name = "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2" +n_layers = 2 d_model = 512 d_mlp = 2048 d_head = 64 @@ -13,12 +13,12 @@ d_vocab = 32000 seed = 10 device = "cuda" use_bfloat16_matmul = false -batch_size_per_device = 16 +batch_size_per_device = 32 n_devices = 1 batches_per_step = 1 max_tokens = 200000000 -lr_hidden = 0.002 -lr_vector = 0.001 +lr_hidden = 0.001 +lr_vector = 0.0005 lr_schedule = "constant_with_warmup" warmup_tokens = 30000000 weight_decay = 0.05 diff --git a/latest_checkpoint.pt b/latest_checkpoint.pt index 10c15aa12aec90d7ca6087993760697228c52ad2..d88ccea2adafa60ecaa187c90732d6dfcaa3968f 100644 --- a/latest_checkpoint.pt +++ b/latest_checkpoint.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1ef12c55e4064f31928e0e94faec665fbe234169ab01323103f9ed95bbb6a5ec -size 284685195 +oid sha256:2d8b6c73ac1d0cb7763633da031d94a3471987c041b9d36454aaf0b81ac6bb36 +size 158534135 diff --git a/latest_metadata.json b/latest_metadata.json index b89a001b93d5048c0cb80071284c2e6c56a8fc4e..fb90620e84e611d9fb437aadda268c8c2a26688f 100644 --- a/latest_metadata.json +++ b/latest_metadata.json @@ -1 +1 @@ -{"step": 12206, "tokens_seen": 199983104, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.1248199771775043} \ No newline at end of file +{"step": 6102, "tokens_seen": 199950336, "config": {"model_name": "pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.001, "lr_vector": 0.0005, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.048876639155322} \ No newline at end of file diff --git a/latest_optimizer.pt b/latest_optimizer.pt index 9748dc269d4f18ccc0dd49397d563ee211ad070a..4f0215a8b09d72bde905fe5ed813b4a55a45d7f0 100644 --- a/latest_optimizer.pt +++ b/latest_optimizer.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:01cc58843d7c9e2b10d7894a42f6d983d75e1d679af4e2cec69a29589944db11 -size 569402163 +oid sha256:8781e8b477df86d8fa0d707933faf109751d929ade9627d653515875e93abde0 +size 317074195 diff --git a/wandb/debug-internal.log b/wandb/debug-internal.log index acf4434179822f6b9a52fc76ffbe4efd3446278f..72c863f7cb04979be388bc3b7d0e5baae06628d6 100644 --- a/wandb/debug-internal.log +++ b/wandb/debug-internal.log @@ -1,12 +1,12 @@ -{"time":"2026-03-19T09:10:54.518147627Z","level":"INFO","msg":"stream: starting","core version":"0.21.4"} -{"time":"2026-03-19T09:10:54.728754949Z","level":"INFO","msg":"stream: created new stream","id":"lisp43b6"} -{"time":"2026-03-19T09:10:54.728795071Z","level":"INFO","msg":"stream: started","id":"lisp43b6"} -{"time":"2026-03-19T09:10:54.729757806Z","level":"INFO","msg":"sender: started","stream_id":"lisp43b6"} -{"time":"2026-03-19T09:10:54.72979291Z","level":"INFO","msg":"handler: started","stream_id":"lisp43b6"} -{"time":"2026-03-19T09:10:54.729814605Z","level":"INFO","msg":"writer: started","stream_id":"lisp43b6"} -{"time":"2026-03-19T12:06:02.518903145Z","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"uploading data","runtime_seconds":9.420394008},{"desc":"updating run metadata","runtime_seconds":0.001821517}],"total_operations":2}} -{"time":"2026-03-19T12:06:03.072131148Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} -{"time":"2026-03-19T12:06:12.889325955Z","level":"INFO","msg":"stream: closing","id":"lisp43b6"} -{"time":"2026-03-19T12:06:12.889384907Z","level":"INFO","msg":"handler: closed","stream_id":"lisp43b6"} -{"time":"2026-03-19T12:06:12.889456031Z","level":"INFO","msg":"sender: closed","stream_id":"lisp43b6"} -{"time":"2026-03-19T12:06:12.889466313Z","level":"INFO","msg":"stream: closed","id":"lisp43b6"} +{"time":"2026-03-25T09:21:22.41420794Z","level":"INFO","msg":"stream: starting","core version":"0.21.4"} +{"time":"2026-03-25T09:21:22.675339572Z","level":"INFO","msg":"stream: created new stream","id":"4guua5vm"} +{"time":"2026-03-25T09:21:22.675405951Z","level":"INFO","msg":"stream: started","id":"4guua5vm"} +{"time":"2026-03-25T09:21:22.675456409Z","level":"INFO","msg":"sender: started","stream_id":"4guua5vm"} +{"time":"2026-03-25T09:21:22.675452087Z","level":"INFO","msg":"writer: started","stream_id":"4guua5vm"} +{"time":"2026-03-25T09:21:22.675491769Z","level":"INFO","msg":"handler: started","stream_id":"4guua5vm"} +{"time":"2026-03-25T10:15:02.459446173Z","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"uploading history steps 241-241, summary, console lines 250-250","runtime_seconds":9.164065763},{"desc":"updating run metadata","runtime_seconds":0.001753079}],"total_operations":2}} +{"time":"2026-03-25T10:15:02.93303388Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-03-25T10:15:12.704963398Z","level":"INFO","msg":"stream: closing","id":"4guua5vm"} +{"time":"2026-03-25T10:15:12.705022861Z","level":"INFO","msg":"handler: closed","stream_id":"4guua5vm"} +{"time":"2026-03-25T10:15:12.705073682Z","level":"INFO","msg":"sender: closed","stream_id":"4guua5vm"} +{"time":"2026-03-25T10:15:12.705122378Z","level":"INFO","msg":"stream: closed","id":"4guua5vm"} diff --git a/wandb/debug.log b/wandb/debug.log index 6341541a524f3bf7b2378d00f86247eec82d17aa..6ec9be4241b6530990520ecbefee02d9f563309c 100644 --- a/wandb/debug.log +++ b/wandb/debug.log @@ -1,26 +1,26 @@ -2026-03-19 09:10:54,307 INFO MainThread:678 [wandb_setup.py:_flush():81] Current SDK version is 0.21.4 -2026-03-19 09:10:54,307 INFO MainThread:678 [wandb_setup.py:_flush():81] Configure stats pid to 678 -2026-03-19 09:10:54,307 INFO MainThread:678 [wandb_setup.py:_flush():81] Loading settings from /root/.config/wandb/settings -2026-03-19 09:10:54,307 INFO MainThread:678 [wandb_setup.py:_flush():81] Loading settings from /notebooks/toy_models/model_training/model/wandb/settings -2026-03-19 09:10:54,307 INFO MainThread:678 [wandb_setup.py:_flush():81] Loading settings from environment variables -2026-03-19 09:10:54,307 INFO MainThread:678 [wandb_init.py:setup_run_log_directory():686] Logging user logs to /notebooks/toy_models/model_training/model/wandb/run-20260319_091054-lisp43b6/logs/debug.log -2026-03-19 09:10:54,307 INFO MainThread:678 [wandb_init.py:setup_run_log_directory():687] Logging internal logs to /notebooks/toy_models/model_training/model/wandb/run-20260319_091054-lisp43b6/logs/debug-internal.log -2026-03-19 09:10:54,307 INFO MainThread:678 [wandb_init.py:init():813] calling init triggers -2026-03-19 09:10:54,307 INFO MainThread:678 [wandb_init.py:init():818] wandb.init called with sweep_config: {} -config: {'model_name': 'pile_llama_replace_17367_L8_1024', 'n_layers': 12, 'd_model': 512, 'd_mlp': 2048, 'd_head': 64, 'n_heads': 8, 'attn_only': False, 'layer_norm_eps': 1e-05, 'init_range': 0.02, 'n_ctx': 1024, 'd_vocab': 32000, 'dataset_name': 'eoinf/PL_Replace17367_L2_alldataset', 'tokenizer_name': '', 'seed': 10, 'data_seed': 10, 'device': 'cuda', 'use_bfloat16_matmul': False, 'batch_size_per_device': 16, 'n_devices': 1, 'batches_per_step': 1, 'max_tokens': 200000000, 'lr_hidden': 0.002, 'lr_vector': 0.001, 'lr_schedule': 'constant_with_warmup', 'warmup_tokens': 30000000, 'weight_decay': 0.05, 'grad_norm_clip': 1.0, 'train_loss_moving_average_beta': 0.99, 'log_interval': 25, 'save_checkpoints': True, 'checkpoint_interval': 500, 'checkpoint_interval_ratio': 1.1, 'save_log_checkpoints': True, 'use_wandb': True, 'batch_size': 16, 'tokens_per_step': 16384, 'warmup_steps': 1831, 'max_steps': 12207, '_wandb': {}} -2026-03-19 09:10:54,307 INFO MainThread:678 [wandb_init.py:init():854] starting backend -2026-03-19 09:10:54,512 INFO MainThread:678 [wandb_init.py:init():857] sending inform_init request -2026-03-19 09:10:54,516 INFO MainThread:678 [wandb_init.py:init():865] backend started and connected -2026-03-19 09:10:54,517 INFO MainThread:678 [wandb_init.py:init():936] updated telemetry -2026-03-19 09:10:54,557 INFO MainThread:678 [wandb_init.py:init():960] communicating run to backend with 90.0 second timeout -2026-03-19 09:10:55,330 INFO MainThread:678 [wandb_init.py:init():1011] starting run threads in backend -2026-03-19 09:10:55,439 INFO MainThread:678 [wandb_run.py:_console_start():2506] atexit reg -2026-03-19 09:10:55,440 INFO MainThread:678 [wandb_run.py:_redirect():2354] redirect: wrap_raw -2026-03-19 09:10:55,440 INFO MainThread:678 [wandb_run.py:_redirect():2423] Wrapping output streams. -2026-03-19 09:10:55,440 INFO MainThread:678 [wandb_run.py:_redirect():2446] Redirects installed. -2026-03-19 09:10:55,442 INFO MainThread:678 [wandb_init.py:init():1049] run started, returning control to user process -2026-03-19 12:06:02,510 INFO MainThread:678 [wandb_run.py:_finish():2272] finishing run tzach/toy-transformer-replication/lisp43b6 -2026-03-19 12:06:02,516 INFO MainThread:678 [wandb_run.py:_atexit_cleanup():2471] got exitcode: 0 -2026-03-19 12:06:02,517 INFO MainThread:678 [wandb_run.py:_restore():2453] restore -2026-03-19 12:06:02,517 INFO MainThread:678 [wandb_run.py:_restore():2459] restore done -2026-03-19 12:06:12,886 INFO MainThread:678 [wandb_run.py:_footer_sync_info():3867] logging synced files +2026-03-25 09:21:21,913 INFO MainThread:3604 [wandb_setup.py:_flush():81] Current SDK version is 0.21.4 +2026-03-25 09:21:21,913 INFO MainThread:3604 [wandb_setup.py:_flush():81] Configure stats pid to 3604 +2026-03-25 09:21:21,913 INFO MainThread:3604 [wandb_setup.py:_flush():81] Loading settings from /root/.config/wandb/settings +2026-03-25 09:21:21,913 INFO MainThread:3604 [wandb_setup.py:_flush():81] Loading settings from /notebooks/toy_models/model_training/model/wandb/settings +2026-03-25 09:21:21,913 INFO MainThread:3604 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-03-25 09:21:21,913 INFO MainThread:3604 [wandb_init.py:setup_run_log_directory():686] Logging user logs to /notebooks/toy_models/model_training/model/wandb/run-20260325_092121-4guua5vm/logs/debug.log +2026-03-25 09:21:21,913 INFO MainThread:3604 [wandb_init.py:setup_run_log_directory():687] Logging internal logs to /notebooks/toy_models/model_training/model/wandb/run-20260325_092121-4guua5vm/logs/debug-internal.log +2026-03-25 09:21:21,913 INFO MainThread:3604 [wandb_init.py:init():813] calling init triggers +2026-03-25 09:21:21,913 INFO MainThread:3604 [wandb_init.py:init():818] wandb.init called with sweep_config: {} +config: {'model_name': 'pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2', 'n_layers': 2, 'd_model': 512, 'd_mlp': 2048, 'd_head': 64, 'n_heads': 8, 'attn_only': False, 'layer_norm_eps': 1e-05, 'init_range': 0.02, 'n_ctx': 1024, 'd_vocab': 32000, 'dataset_name': 'eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2', 'tokenizer_name': '', 'seed': 10, 'data_seed': 10, 'device': 'cuda', 'use_bfloat16_matmul': False, 'batch_size_per_device': 32, 'n_devices': 1, 'batches_per_step': 1, 'max_tokens': 200000000, 'lr_hidden': 0.001, 'lr_vector': 0.0005, 'lr_schedule': 'constant_with_warmup', 'warmup_tokens': 30000000, 'weight_decay': 0.05, 'grad_norm_clip': 1.0, 'train_loss_moving_average_beta': 0.99, 'log_interval': 25, 'save_checkpoints': True, 'checkpoint_interval': 500, 'checkpoint_interval_ratio': 1.1, 'save_log_checkpoints': True, 'use_wandb': True, 'batch_size': 32, 'tokens_per_step': 32768, 'warmup_steps': 915, 'max_steps': 6103, '_wandb': {}} +2026-03-25 09:21:21,913 INFO MainThread:3604 [wandb_init.py:init():854] starting backend +2026-03-25 09:21:22,395 INFO MainThread:3604 [wandb_init.py:init():857] sending inform_init request +2026-03-25 09:21:22,409 INFO MainThread:3604 [wandb_init.py:init():865] backend started and connected +2026-03-25 09:21:22,411 INFO MainThread:3604 [wandb_init.py:init():936] updated telemetry +2026-03-25 09:21:22,417 INFO MainThread:3604 [wandb_init.py:init():960] communicating run to backend with 90.0 second timeout +2026-03-25 09:21:23,291 INFO MainThread:3604 [wandb_init.py:init():1011] starting run threads in backend +2026-03-25 09:21:24,185 INFO MainThread:3604 [wandb_run.py:_console_start():2506] atexit reg +2026-03-25 09:21:24,186 INFO MainThread:3604 [wandb_run.py:_redirect():2354] redirect: wrap_raw +2026-03-25 09:21:24,186 INFO MainThread:3604 [wandb_run.py:_redirect():2423] Wrapping output streams. +2026-03-25 09:21:24,186 INFO MainThread:3604 [wandb_run.py:_redirect():2446] Redirects installed. +2026-03-25 09:21:24,197 INFO MainThread:3604 [wandb_init.py:init():1049] run started, returning control to user process +2026-03-25 10:15:02,444 INFO MainThread:3604 [wandb_run.py:_finish():2272] finishing run tzach/toy-transformer-replication/4guua5vm +2026-03-25 10:15:02,456 INFO MainThread:3604 [wandb_run.py:_atexit_cleanup():2471] got exitcode: 0 +2026-03-25 10:15:02,457 INFO MainThread:3604 [wandb_run.py:_restore():2453] restore +2026-03-25 10:15:02,457 INFO MainThread:3604 [wandb_run.py:_restore():2459] restore done +2026-03-25 10:15:12,704 INFO MainThread:3604 [wandb_run.py:_footer_sync_info():3867] logging synced files diff --git a/wandb/run-20260325_092121-4guua5vm/files/config.yaml b/wandb/run-20260325_092121-4guua5vm/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb3ad88837f7ac8ba1aff6aeb248544bada3aac7 --- /dev/null +++ b/wandb/run-20260325_092121-4guua5vm/files/config.yaml @@ -0,0 +1,140 @@ +_wandb: + value: + cli_version: 0.21.4 + e: + sflf975jycouqau4pywajxey1ffzx7ln: + cpu_count: 8 + cpu_count_logical: 8 + cudaVersion: "12.4" + disk: + /: + total: "262240792576" + used: "121270431744" + email: tzfof8@gmail.com + executable: /notebooks/toy_models/.toy_models_env/bin/python + git: + commit: 6d40e57307f8f7c2c410c6d3dac053261e8a836d + remote: https://github.com/jgroh3/toy_models.git + gpu: NVIDIA RTX A6000 + gpu_count: 1 + gpu_nvidia: + - architecture: Ampere + cudaCores: 10752 + memoryTotal: "51527024640" + name: NVIDIA RTX A6000 + uuid: GPU-dabc88f0-4b6a-64c4-f799-f3f38a477a96 + host: nwlkget26z + memory: + total: "47332843520" + os: Linux-5.19.0-45-generic-x86_64-with-glibc2.35 + program: + python: CPython 3.11.7 + root: /notebooks/toy_models/model_training/model + startedAt: "2026-03-25T09:21:21.907528Z" + writerId: sflf975jycouqau4pywajxey1ffzx7ln + m: [] + python_version: 3.11.7 + t: + "1": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 71 + "2": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 71 + "3": + - 2 + - 13 + - 15 + - 16 + - 61 + "4": 3.11.7 + "5": 0.21.4 + "6": 4.56.1 + "12": 0.21.4 + "13": linux-x86_64 +attn_only: + value: false +batch_size: + value: 32 +batch_size_per_device: + value: 32 +batches_per_step: + value: 1 +checkpoint_interval: + value: 500 +checkpoint_interval_ratio: + value: 1.1 +d_head: + value: 64 +d_mlp: + value: 2048 +d_model: + value: 512 +d_vocab: + value: 32000 +data_seed: + value: 10 +dataset_name: + value: eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2 +device: + value: cuda +grad_norm_clip: + value: 1 +init_range: + value: 0.02 +layer_norm_eps: + value: 1e-05 +log_interval: + value: 25 +lr_hidden: + value: 0.001 +lr_schedule: + value: constant_with_warmup +lr_vector: + value: 0.0005 +max_steps: + value: 6103 +max_tokens: + value: 200000000 +model_name: + value: pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2 +n_ctx: + value: 1024 +n_devices: + value: 1 +n_heads: + value: 8 +n_layers: + value: 2 +save_checkpoints: + value: true +save_log_checkpoints: + value: true +seed: + value: 10 +tokenizer_name: + value: "" +tokens_per_step: + value: 32768 +train_loss_moving_average_beta: + value: 0.99 +use_bfloat16_matmul: + value: false +use_wandb: + value: true +warmup_steps: + value: 915 +warmup_tokens: + value: 30000000 +weight_decay: + value: 0.05 diff --git a/wandb/run-20260325_092121-4guua5vm/files/output.log b/wandb/run-20260325_092121-4guua5vm/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..6ca5a4f68e87f0aecc251c179ae524d6c015be50 --- /dev/null +++ b/wandb/run-20260325_092121-4guua5vm/files/output.log @@ -0,0 +1,252 @@ +Training on cuda +Model: 2L, 512d, 8h +Max steps: 6,103, Max tokens: 200,000,000 +Warmup steps: 915, Warmup tokens: 30,000,000 +Batch size per device: 32 +Context length: 1024 +Learning rates - Hidden: 0.001, Vector: 0.0005 + +Step 25 | Tokens: 819,200 | Train Loss EWMA: 10.4493 | Learning Rate: 0.000027 | Progress: 0.00410 +Step 50 | Tokens: 1,638,400 | Train Loss EWMA: 10.2914 | Learning Rate: 0.000055 | Progress: 0.00819 +Step 75 | Tokens: 2,457,600 | Train Loss EWMA: 10.0750 | Learning Rate: 0.000082 | Progress: 0.01229 +Step 100 | Tokens: 3,276,800 | Train Loss EWMA: 9.7935 | Learning Rate: 0.000109 | Progress: 0.01638 +Step 125 | Tokens: 4,096,000 | Train Loss EWMA: 9.4639 | Learning Rate: 0.000137 | Progress: 0.02048 +Step 150 | Tokens: 4,915,200 | Train Loss EWMA: 9.1289 | Learning Rate: 0.000164 | Progress: 0.02458 +Step 175 | Tokens: 5,734,400 | Train Loss EWMA: 8.8396 | Learning Rate: 0.000191 | Progress: 0.02867 +Step 200 | Tokens: 6,553,600 | Train Loss EWMA: 8.5954 | Learning Rate: 0.000219 | Progress: 0.03277 +Step 225 | Tokens: 7,372,800 | Train Loss EWMA: 8.3861 | Learning Rate: 0.000246 | Progress: 0.03686 +Step 250 | Tokens: 8,192,000 | Train Loss EWMA: 8.2158 | Learning Rate: 0.000273 | Progress: 0.04096 +Step 275 | Tokens: 9,011,200 | Train Loss EWMA: 8.0639 | Learning Rate: 0.000301 | Progress: 0.04506 +Step 300 | Tokens: 9,830,400 | Train Loss EWMA: 7.9306 | Learning Rate: 0.000328 | Progress: 0.04915 +Step 325 | Tokens: 10,649,600 | Train Loss EWMA: 7.8169 | Learning Rate: 0.000355 | Progress: 0.05325 +Step 350 | Tokens: 11,468,800 | Train Loss EWMA: 7.7209 | Learning Rate: 0.000383 | Progress: 0.05734 +Step 375 | Tokens: 12,288,000 | Train Loss EWMA: 7.6356 | Learning Rate: 0.000410 | Progress: 0.06144 +Step 400 | Tokens: 13,107,200 | Train Loss EWMA: 7.5605 | Learning Rate: 0.000437 | Progress: 0.06554 +Step 425 | Tokens: 13,926,400 | Train Loss EWMA: 7.4906 | Learning Rate: 0.000464 | Progress: 0.06963 +Step 450 | Tokens: 14,745,600 | Train Loss EWMA: 7.4341 | Learning Rate: 0.000492 | Progress: 0.07373 +Step 475 | Tokens: 15,564,800 | Train Loss EWMA: 7.3797 | Learning Rate: 0.000519 | Progress: 0.07782 +Step 500 | Tokens: 16,384,000 | Train Loss EWMA: 7.3387 | Learning Rate: 0.000546 | Progress: 0.08192 +Step 525 | Tokens: 17,203,200 | Train Loss EWMA: 7.2938 | Learning Rate: 0.000574 | Progress: 0.08602 +Step 550 | Tokens: 18,022,400 | Train Loss EWMA: 7.2514 | Learning Rate: 0.000601 | Progress: 0.09011 +Step 575 | Tokens: 18,841,600 | Train Loss EWMA: 7.2098 | Learning Rate: 0.000628 | Progress: 0.09421 +Step 600 | Tokens: 19,660,800 | Train Loss EWMA: 7.1784 | Learning Rate: 0.000656 | Progress: 0.09830 +Step 625 | Tokens: 20,480,000 | Train Loss EWMA: 7.1482 | Learning Rate: 0.000683 | Progress: 0.10240 +Step 650 | Tokens: 21,299,200 | Train Loss EWMA: 7.1286 | Learning Rate: 0.000710 | Progress: 0.10650 +Step 675 | Tokens: 22,118,400 | Train Loss EWMA: 7.1012 | Learning Rate: 0.000738 | Progress: 0.11059 +Step 700 | Tokens: 22,937,600 | Train Loss EWMA: 7.0781 | Learning Rate: 0.000765 | Progress: 0.11469 +Step 725 | Tokens: 23,756,800 | Train Loss EWMA: 7.0515 | Learning Rate: 0.000792 | Progress: 0.11878 +Step 750 | Tokens: 24,576,000 | Train Loss EWMA: 7.0220 | Learning Rate: 0.000820 | Progress: 0.12288 +Step 775 | Tokens: 25,395,200 | Train Loss EWMA: 7.0039 | Learning Rate: 0.000847 | Progress: 0.12698 +Step 800 | Tokens: 26,214,400 | Train Loss EWMA: 6.9874 | Learning Rate: 0.000874 | Progress: 0.13107 +Step 825 | Tokens: 27,033,600 | Train Loss EWMA: 6.9662 | Learning Rate: 0.000902 | Progress: 0.13517 +Step 850 | Tokens: 27,852,800 | Train Loss EWMA: 6.9477 | Learning Rate: 0.000929 | Progress: 0.13926 +Step 875 | Tokens: 28,672,000 | Train Loss EWMA: 6.9271 | Learning Rate: 0.000956 | Progress: 0.14336 +Step 900 | Tokens: 29,491,200 | Train Loss EWMA: 6.9098 | Learning Rate: 0.000984 | Progress: 0.14746 +Step 925 | Tokens: 30,310,400 | Train Loss EWMA: 6.8862 | Learning Rate: 0.001000 | Progress: 0.15155 +Step 950 | Tokens: 31,129,600 | Train Loss EWMA: 6.8727 | Learning Rate: 0.001000 | Progress: 0.15565 +Step 975 | Tokens: 31,948,800 | Train Loss EWMA: 6.8587 | Learning Rate: 0.001000 | Progress: 0.15974 +Step 1,000 | Tokens: 32,768,000 | Train Loss EWMA: 6.8403 | Learning Rate: 0.001000 | Progress: 0.16384 +Step 1,025 | Tokens: 33,587,200 | Train Loss EWMA: 6.8233 | Learning Rate: 0.001000 | Progress: 0.16794 +Step 1,050 | Tokens: 34,406,400 | Train Loss EWMA: 6.8103 | Learning Rate: 0.001000 | Progress: 0.17203 +Step 1,075 | Tokens: 35,225,600 | Train Loss EWMA: 6.7965 | Learning Rate: 0.001000 | Progress: 0.17613 +Step 1,100 | Tokens: 36,044,800 | Train Loss EWMA: 6.7812 | Learning Rate: 0.001000 | Progress: 0.18022 +Step 1,125 | Tokens: 36,864,000 | Train Loss EWMA: 6.7667 | Learning Rate: 0.001000 | Progress: 0.18432 +Step 1,150 | Tokens: 37,683,200 | Train Loss EWMA: 6.7552 | Learning Rate: 0.001000 | Progress: 0.18842 +Step 1,175 | Tokens: 38,502,400 | Train Loss EWMA: 6.7402 | Learning Rate: 0.001000 | Progress: 0.19251 +Step 1,200 | Tokens: 39,321,600 | Train Loss EWMA: 6.7296 | Learning Rate: 0.001000 | Progress: 0.19661 +Step 1,225 | Tokens: 40,140,800 | Train Loss EWMA: 6.7185 | Learning Rate: 0.001000 | Progress: 0.20070 +Step 1,250 | Tokens: 40,960,000 | Train Loss EWMA: 6.7015 | Learning Rate: 0.001000 | Progress: 0.20480 +Step 1,275 | Tokens: 41,779,200 | Train Loss EWMA: 6.6888 | Learning Rate: 0.001000 | Progress: 0.20890 +Step 1,300 | Tokens: 42,598,400 | Train Loss EWMA: 6.6765 | Learning Rate: 0.001000 | Progress: 0.21299 +Step 1,325 | Tokens: 43,417,600 | Train Loss EWMA: 6.6716 | Learning Rate: 0.001000 | Progress: 0.21709 +Step 1,350 | Tokens: 44,236,800 | Train Loss EWMA: 6.6544 | Learning Rate: 0.001000 | Progress: 0.22118 +Step 1,375 | Tokens: 45,056,000 | Train Loss EWMA: 6.6406 | Learning Rate: 0.001000 | Progress: 0.22528 +Step 1,400 | Tokens: 45,875,200 | Train Loss EWMA: 6.6312 | Learning Rate: 0.001000 | Progress: 0.22938 +Step 1,425 | Tokens: 46,694,400 | Train Loss EWMA: 6.6221 | Learning Rate: 0.001000 | Progress: 0.23347 +Step 1,450 | Tokens: 47,513,600 | Train Loss EWMA: 6.6150 | Learning Rate: 0.001000 | Progress: 0.23757 +Step 1,475 | Tokens: 48,332,800 | Train Loss EWMA: 6.6019 | Learning Rate: 0.001000 | Progress: 0.24166 +Step 1,500 | Tokens: 49,152,000 | Train Loss EWMA: 6.5938 | Learning Rate: 0.001000 | Progress: 0.24576 +Step 1,525 | Tokens: 49,971,200 | Train Loss EWMA: 6.5918 | Learning Rate: 0.001000 | Progress: 0.24986 +Step 1,550 | Tokens: 50,790,400 | Train Loss EWMA: 6.5801 | Learning Rate: 0.001000 | Progress: 0.25395 +Step 1,575 | Tokens: 51,609,600 | Train Loss EWMA: 6.5675 | Learning Rate: 0.001000 | Progress: 0.25805 +Step 1,600 | Tokens: 52,428,800 | Train Loss EWMA: 6.5579 | Learning Rate: 0.001000 | Progress: 0.26214 +Step 1,625 | Tokens: 53,248,000 | Train Loss EWMA: 6.5476 | Learning Rate: 0.001000 | Progress: 0.26624 +Step 1,650 | Tokens: 54,067,200 | Train Loss EWMA: 6.5415 | Learning Rate: 0.001000 | Progress: 0.27034 +Step 1,675 | Tokens: 54,886,400 | Train Loss EWMA: 6.5278 | Learning Rate: 0.001000 | Progress: 0.27443 +Step 1,700 | Tokens: 55,705,600 | Train Loss EWMA: 6.5126 | Learning Rate: 0.001000 | Progress: 0.27853 +Step 1,725 | Tokens: 56,524,800 | Train Loss EWMA: 6.5072 | Learning Rate: 0.001000 | Progress: 0.28262 +Step 1,750 | Tokens: 57,344,000 | Train Loss EWMA: 6.4973 | Learning Rate: 0.001000 | Progress: 0.28672 +Step 1,775 | Tokens: 58,163,200 | Train Loss EWMA: 6.4883 | Learning Rate: 0.001000 | Progress: 0.29082 +Step 1,800 | Tokens: 58,982,400 | Train Loss EWMA: 6.4906 | Learning Rate: 0.001000 | Progress: 0.29491 +Step 1,825 | Tokens: 59,801,600 | Train Loss EWMA: 6.4789 | Learning Rate: 0.001000 | Progress: 0.29901 +Step 1,850 | Tokens: 60,620,800 | Train Loss EWMA: 6.4715 | Learning Rate: 0.001000 | Progress: 0.30310 +Step 1,875 | Tokens: 61,440,000 | Train Loss EWMA: 6.4668 | Learning Rate: 0.001000 | Progress: 0.30720 +Step 1,900 | Tokens: 62,259,200 | Train Loss EWMA: 6.4642 | Learning Rate: 0.001000 | Progress: 0.31130 +Step 1,925 | Tokens: 63,078,400 | Train Loss EWMA: 6.4672 | Learning Rate: 0.001000 | Progress: 0.31539 +Step 1,950 | Tokens: 63,897,600 | Train Loss EWMA: 6.4638 | Learning Rate: 0.001000 | Progress: 0.31949 +Step 1,975 | Tokens: 64,716,800 | Train Loss EWMA: 6.4617 | Learning Rate: 0.001000 | Progress: 0.32358 +Step 2,000 | Tokens: 65,536,000 | Train Loss EWMA: 6.4552 | Learning Rate: 0.001000 | Progress: 0.32768 +Step 2,025 | Tokens: 66,355,200 | Train Loss EWMA: 6.4473 | Learning Rate: 0.001000 | Progress: 0.33178 +Step 2,050 | Tokens: 67,174,400 | Train Loss EWMA: 6.4431 | Learning Rate: 0.001000 | Progress: 0.33587 +Step 2,075 | Tokens: 67,993,600 | Train Loss EWMA: 6.4296 | Learning Rate: 0.001000 | Progress: 0.33997 +Step 2,100 | Tokens: 68,812,800 | Train Loss EWMA: 6.4180 | Learning Rate: 0.001000 | Progress: 0.34406 +Step 2,125 | Tokens: 69,632,000 | Train Loss EWMA: 6.4143 | Learning Rate: 0.001000 | Progress: 0.34816 +Step 2,150 | Tokens: 70,451,200 | Train Loss EWMA: 6.4142 | Learning Rate: 0.001000 | Progress: 0.35226 +Step 2,175 | Tokens: 71,270,400 | Train Loss EWMA: 6.4074 | Learning Rate: 0.001000 | Progress: 0.35635 +Step 2,200 | Tokens: 72,089,600 | Train Loss EWMA: 6.4051 | Learning Rate: 0.001000 | Progress: 0.36045 +Step 2,225 | Tokens: 72,908,800 | Train Loss EWMA: 6.3977 | Learning Rate: 0.001000 | Progress: 0.36454 +Step 2,250 | Tokens: 73,728,000 | Train Loss EWMA: 6.3928 | Learning Rate: 0.001000 | Progress: 0.36864 +Step 2,275 | Tokens: 74,547,200 | Train Loss EWMA: 6.3896 | Learning Rate: 0.001000 | Progress: 0.37274 +Step 2,300 | Tokens: 75,366,400 | Train Loss EWMA: 6.3836 | Learning Rate: 0.001000 | Progress: 0.37683 +Step 2,325 | Tokens: 76,185,600 | Train Loss EWMA: 6.3777 | Learning Rate: 0.001000 | Progress: 0.38093 +Step 2,350 | Tokens: 77,004,800 | Train Loss EWMA: 6.3751 | Learning Rate: 0.001000 | Progress: 0.38502 +Step 2,375 | Tokens: 77,824,000 | Train Loss EWMA: 6.3695 | Learning Rate: 0.001000 | Progress: 0.38912 +Step 2,400 | Tokens: 78,643,200 | Train Loss EWMA: 6.3640 | Learning Rate: 0.001000 | Progress: 0.39322 +Step 2,425 | Tokens: 79,462,400 | Train Loss EWMA: 6.3599 | Learning Rate: 0.001000 | Progress: 0.39731 +Step 2,450 | Tokens: 80,281,600 | Train Loss EWMA: 6.3622 | Learning Rate: 0.001000 | Progress: 0.40141 +Step 2,475 | Tokens: 81,100,800 | Train Loss EWMA: 6.3572 | Learning Rate: 0.001000 | Progress: 0.40550 +Step 2,500 | Tokens: 81,920,000 | Train Loss EWMA: 6.3510 | Learning Rate: 0.001000 | Progress: 0.40960 +Step 2,525 | Tokens: 82,739,200 | Train Loss EWMA: 6.3435 | Learning Rate: 0.001000 | Progress: 0.41370 +Step 2,550 | Tokens: 83,558,400 | Train Loss EWMA: 6.3417 | Learning Rate: 0.001000 | Progress: 0.41779 +Step 2,575 | Tokens: 84,377,600 | Train Loss EWMA: 6.3326 | Learning Rate: 0.001000 | Progress: 0.42189 +Step 2,600 | Tokens: 85,196,800 | Train Loss EWMA: 6.3272 | Learning Rate: 0.001000 | Progress: 0.42598 +Step 2,625 | Tokens: 86,016,000 | Train Loss EWMA: 6.3271 | Learning Rate: 0.001000 | Progress: 0.43008 +Step 2,650 | Tokens: 86,835,200 | Train Loss EWMA: 6.3227 | Learning Rate: 0.001000 | Progress: 0.43418 +Step 2,675 | Tokens: 87,654,400 | Train Loss EWMA: 6.3206 | Learning Rate: 0.001000 | Progress: 0.43827 +Step 2,700 | Tokens: 88,473,600 | Train Loss EWMA: 6.3101 | Learning Rate: 0.001000 | Progress: 0.44237 +Step 2,725 | Tokens: 89,292,800 | Train Loss EWMA: 6.3111 | Learning Rate: 0.001000 | Progress: 0.44646 +Step 2,750 | Tokens: 90,112,000 | Train Loss EWMA: 6.3043 | Learning Rate: 0.001000 | Progress: 0.45056 +Step 2,775 | Tokens: 90,931,200 | Train Loss EWMA: 6.3021 | Learning Rate: 0.001000 | Progress: 0.45466 +Step 2,800 | Tokens: 91,750,400 | Train Loss EWMA: 6.2983 | Learning Rate: 0.001000 | Progress: 0.45875 +Step 2,825 | Tokens: 92,569,600 | Train Loss EWMA: 6.2991 | Learning Rate: 0.001000 | Progress: 0.46285 +Step 2,850 | Tokens: 93,388,800 | Train Loss EWMA: 6.2960 | Learning Rate: 0.001000 | Progress: 0.46694 +Step 2,875 | Tokens: 94,208,000 | Train Loss EWMA: 6.2877 | Learning Rate: 0.001000 | Progress: 0.47104 +Step 2,900 | Tokens: 95,027,200 | Train Loss EWMA: 6.2839 | Learning Rate: 0.001000 | Progress: 0.47514 +Step 2,925 | Tokens: 95,846,400 | Train Loss EWMA: 6.2811 | Learning Rate: 0.001000 | Progress: 0.47923 +Step 2,950 | Tokens: 96,665,600 | Train Loss EWMA: 6.2787 | Learning Rate: 0.001000 | Progress: 0.48333 +Step 2,975 | Tokens: 97,484,800 | Train Loss EWMA: 6.2831 | Learning Rate: 0.001000 | Progress: 0.48742 +Step 3,000 | Tokens: 98,304,000 | Train Loss EWMA: 6.2824 | Learning Rate: 0.001000 | Progress: 0.49152 +Step 3,025 | Tokens: 99,123,200 | Train Loss EWMA: 6.2771 | Learning Rate: 0.001000 | Progress: 0.49562 +Step 3,050 | Tokens: 99,942,400 | Train Loss EWMA: 6.2713 | Learning Rate: 0.001000 | Progress: 0.49971 +Step 3,075 | Tokens: 100,761,600 | Train Loss EWMA: 6.2671 | Learning Rate: 0.001000 | Progress: 0.50381 +Step 3,100 | Tokens: 101,580,800 | Train Loss EWMA: 6.2653 | Learning Rate: 0.001000 | Progress: 0.50790 +Step 3,125 | Tokens: 102,400,000 | Train Loss EWMA: 6.2624 | Learning Rate: 0.001000 | Progress: 0.51200 +Step 3,150 | Tokens: 103,219,200 | Train Loss EWMA: 6.2578 | Learning Rate: 0.001000 | Progress: 0.51610 +Step 3,175 | Tokens: 104,038,400 | Train Loss EWMA: 6.2545 | Learning Rate: 0.001000 | Progress: 0.52019 +Step 3,200 | Tokens: 104,857,600 | Train Loss EWMA: 6.2477 | Learning Rate: 0.001000 | Progress: 0.52429 +Step 3,225 | Tokens: 105,676,800 | Train Loss EWMA: 6.2444 | Learning Rate: 0.001000 | Progress: 0.52838 +Step 3,250 | Tokens: 106,496,000 | Train Loss EWMA: 6.2428 | Learning Rate: 0.001000 | Progress: 0.53248 +Step 3,275 | Tokens: 107,315,200 | Train Loss EWMA: 6.2452 | Learning Rate: 0.001000 | Progress: 0.53658 +Step 3,300 | Tokens: 108,134,400 | Train Loss EWMA: 6.2393 | Learning Rate: 0.001000 | Progress: 0.54067 +Step 3,325 | Tokens: 108,953,600 | Train Loss EWMA: 6.2367 | Learning Rate: 0.001000 | Progress: 0.54477 +Step 3,350 | Tokens: 109,772,800 | Train Loss EWMA: 6.2383 | Learning Rate: 0.001000 | Progress: 0.54886 +Step 3,375 | Tokens: 110,592,000 | Train Loss EWMA: 6.2286 | Learning Rate: 0.001000 | Progress: 0.55296 +Step 3,400 | Tokens: 111,411,200 | Train Loss EWMA: 6.2268 | Learning Rate: 0.001000 | Progress: 0.55706 +Step 3,425 | Tokens: 112,230,400 | Train Loss EWMA: 6.2206 | Learning Rate: 0.001000 | Progress: 0.56115 +Step 3,450 | Tokens: 113,049,600 | Train Loss EWMA: 6.2199 | Learning Rate: 0.001000 | Progress: 0.56525 +Step 3,475 | Tokens: 113,868,800 | Train Loss EWMA: 6.2245 | Learning Rate: 0.001000 | Progress: 0.56934 +Step 3,500 | Tokens: 114,688,000 | Train Loss EWMA: 6.2155 | Learning Rate: 0.001000 | Progress: 0.57344 +Step 3,525 | Tokens: 115,507,200 | Train Loss EWMA: 6.2138 | Learning Rate: 0.001000 | Progress: 0.57754 +Step 3,550 | Tokens: 116,326,400 | Train Loss EWMA: 6.2076 | Learning Rate: 0.001000 | Progress: 0.58163 +Step 3,575 | Tokens: 117,145,600 | Train Loss EWMA: 6.2154 | Learning Rate: 0.001000 | Progress: 0.58573 +Step 3,600 | Tokens: 117,964,800 | Train Loss EWMA: 6.2127 | Learning Rate: 0.001000 | Progress: 0.58982 +Step 3,625 | Tokens: 118,784,000 | Train Loss EWMA: 6.2154 | Learning Rate: 0.001000 | Progress: 0.59392 +Step 3,650 | Tokens: 119,603,200 | Train Loss EWMA: 6.2134 | Learning Rate: 0.001000 | Progress: 0.59802 +Step 3,675 | Tokens: 120,422,400 | Train Loss EWMA: 6.2127 | Learning Rate: 0.001000 | Progress: 0.60211 +Step 3,700 | Tokens: 121,241,600 | Train Loss EWMA: 6.2096 | Learning Rate: 0.001000 | Progress: 0.60621 +Step 3,725 | Tokens: 122,060,800 | Train Loss EWMA: 6.2065 | Learning Rate: 0.001000 | Progress: 0.61030 +Step 3,750 | Tokens: 122,880,000 | Train Loss EWMA: 6.2007 | Learning Rate: 0.001000 | Progress: 0.61440 +Step 3,775 | Tokens: 123,699,200 | Train Loss EWMA: 6.1915 | Learning Rate: 0.001000 | Progress: 0.61850 +Step 3,800 | Tokens: 124,518,400 | Train Loss EWMA: 6.1873 | Learning Rate: 0.001000 | Progress: 0.62259 +Step 3,825 | Tokens: 125,337,600 | Train Loss EWMA: 6.1861 | Learning Rate: 0.001000 | Progress: 0.62669 +Step 3,850 | Tokens: 126,156,800 | Train Loss EWMA: 6.1907 | Learning Rate: 0.001000 | Progress: 0.63078 +Step 3,875 | Tokens: 126,976,000 | Train Loss EWMA: 6.1875 | Learning Rate: 0.001000 | Progress: 0.63488 +Step 3,900 | Tokens: 127,795,200 | Train Loss EWMA: 6.1931 | Learning Rate: 0.001000 | Progress: 0.63898 +Step 3,925 | Tokens: 128,614,400 | Train Loss EWMA: 6.1881 | Learning Rate: 0.001000 | Progress: 0.64307 +Step 3,950 | Tokens: 129,433,600 | Train Loss EWMA: 6.1895 | Learning Rate: 0.001000 | Progress: 0.64717 +Step 3,975 | Tokens: 130,252,800 | Train Loss EWMA: 6.1819 | Learning Rate: 0.001000 | Progress: 0.65126 +Step 4,000 | Tokens: 131,072,000 | Train Loss EWMA: 6.1852 | Learning Rate: 0.001000 | Progress: 0.65536 +Step 4,025 | Tokens: 131,891,200 | Train Loss EWMA: 6.1812 | Learning Rate: 0.001000 | Progress: 0.65946 +Step 4,050 | Tokens: 132,710,400 | Train Loss EWMA: 6.1700 | Learning Rate: 0.001000 | Progress: 0.66355 +Step 4,075 | Tokens: 133,529,600 | Train Loss EWMA: 6.1678 | Learning Rate: 0.001000 | Progress: 0.66765 +Step 4,100 | Tokens: 134,348,800 | Train Loss EWMA: 6.1626 | Learning Rate: 0.001000 | Progress: 0.67174 +Step 4,125 | Tokens: 135,168,000 | Train Loss EWMA: 6.1660 | Learning Rate: 0.001000 | Progress: 0.67584 +Step 4,150 | Tokens: 135,987,200 | Train Loss EWMA: 6.1677 | Learning Rate: 0.001000 | Progress: 0.67994 +Step 4,175 | Tokens: 136,806,400 | Train Loss EWMA: 6.1657 | Learning Rate: 0.001000 | Progress: 0.68403 +Step 4,200 | Tokens: 137,625,600 | Train Loss EWMA: 6.1735 | Learning Rate: 0.001000 | Progress: 0.68813 +Step 4,225 | Tokens: 138,444,800 | Train Loss EWMA: 6.1678 | Learning Rate: 0.001000 | Progress: 0.69222 +Step 4,250 | Tokens: 139,264,000 | Train Loss EWMA: 6.1669 | Learning Rate: 0.001000 | Progress: 0.69632 +Step 4,275 | Tokens: 140,083,200 | Train Loss EWMA: 6.1652 | Learning Rate: 0.001000 | Progress: 0.70042 +Step 4,300 | Tokens: 140,902,400 | Train Loss EWMA: 6.1555 | Learning Rate: 0.001000 | Progress: 0.70451 +Step 4,325 | Tokens: 141,721,600 | Train Loss EWMA: 6.1539 | Learning Rate: 0.001000 | Progress: 0.70861 +Step 4,350 | Tokens: 142,540,800 | Train Loss EWMA: 6.1507 | Learning Rate: 0.001000 | Progress: 0.71270 +Step 4,375 | Tokens: 143,360,000 | Train Loss EWMA: 6.1521 | Learning Rate: 0.001000 | Progress: 0.71680 +Step 4,400 | Tokens: 144,179,200 | Train Loss EWMA: 6.1570 | Learning Rate: 0.001000 | Progress: 0.72090 +Step 4,425 | Tokens: 144,998,400 | Train Loss EWMA: 6.1610 | Learning Rate: 0.001000 | Progress: 0.72499 +Step 4,450 | Tokens: 145,817,600 | Train Loss EWMA: 6.1637 | Learning Rate: 0.001000 | Progress: 0.72909 +Step 4,475 | Tokens: 146,636,800 | Train Loss EWMA: 6.1620 | Learning Rate: 0.001000 | Progress: 0.73318 +Step 4,500 | Tokens: 147,456,000 | Train Loss EWMA: 6.1493 | Learning Rate: 0.001000 | Progress: 0.73728 +Step 4,525 | Tokens: 148,275,200 | Train Loss EWMA: 6.1393 | Learning Rate: 0.001000 | Progress: 0.74138 +Step 4,550 | Tokens: 149,094,400 | Train Loss EWMA: 6.1396 | Learning Rate: 0.001000 | Progress: 0.74547 +Step 4,575 | Tokens: 149,913,600 | Train Loss EWMA: 6.1332 | Learning Rate: 0.001000 | Progress: 0.74957 +Step 4,600 | Tokens: 150,732,800 | Train Loss EWMA: 6.1349 | Learning Rate: 0.001000 | Progress: 0.75366 +Step 4,625 | Tokens: 151,552,000 | Train Loss EWMA: 6.1307 | Learning Rate: 0.001000 | Progress: 0.75776 +Step 4,650 | Tokens: 152,371,200 | Train Loss EWMA: 6.1252 | Learning Rate: 0.001000 | Progress: 0.76186 +Step 4,675 | Tokens: 153,190,400 | Train Loss EWMA: 6.1215 | Learning Rate: 0.001000 | Progress: 0.76595 +Step 4,700 | Tokens: 154,009,600 | Train Loss EWMA: 6.1236 | Learning Rate: 0.001000 | Progress: 0.77005 +Step 4,725 | Tokens: 154,828,800 | Train Loss EWMA: 6.1264 | Learning Rate: 0.001000 | Progress: 0.77414 +Step 4,750 | Tokens: 155,648,000 | Train Loss EWMA: 6.1284 | Learning Rate: 0.001000 | Progress: 0.77824 +Step 4,775 | Tokens: 156,467,200 | Train Loss EWMA: 6.1287 | Learning Rate: 0.001000 | Progress: 0.78234 +Step 4,800 | Tokens: 157,286,400 | Train Loss EWMA: 6.1311 | Learning Rate: 0.001000 | Progress: 0.78643 +Step 4,825 | Tokens: 158,105,600 | Train Loss EWMA: 6.1276 | Learning Rate: 0.001000 | Progress: 0.79053 +Step 4,850 | Tokens: 158,924,800 | Train Loss EWMA: 6.1268 | Learning Rate: 0.001000 | Progress: 0.79462 +Step 4,875 | Tokens: 159,744,000 | Train Loss EWMA: 6.1269 | Learning Rate: 0.001000 | Progress: 0.79872 +Step 4,900 | Tokens: 160,563,200 | Train Loss EWMA: 6.1260 | Learning Rate: 0.001000 | Progress: 0.80282 +Step 4,925 | Tokens: 161,382,400 | Train Loss EWMA: 6.1112 | Learning Rate: 0.001000 | Progress: 0.80691 +Step 4,950 | Tokens: 162,201,600 | Train Loss EWMA: 6.1131 | Learning Rate: 0.001000 | Progress: 0.81101 +Step 4,975 | Tokens: 163,020,800 | Train Loss EWMA: 6.1151 | Learning Rate: 0.001000 | Progress: 0.81510 +Step 5,000 | Tokens: 163,840,000 | Train Loss EWMA: 6.1090 | Learning Rate: 0.001000 | Progress: 0.81920 +Step 5,025 | Tokens: 164,659,200 | Train Loss EWMA: 6.1129 | Learning Rate: 0.001000 | Progress: 0.82330 +Step 5,050 | Tokens: 165,478,400 | Train Loss EWMA: 6.1085 | Learning Rate: 0.001000 | Progress: 0.82739 +Step 5,075 | Tokens: 166,297,600 | Train Loss EWMA: 6.1123 | Learning Rate: 0.001000 | Progress: 0.83149 +Step 5,100 | Tokens: 167,116,800 | Train Loss EWMA: 6.1053 | Learning Rate: 0.001000 | Progress: 0.83558 +Step 5,125 | Tokens: 167,936,000 | Train Loss EWMA: 6.1001 | Learning Rate: 0.001000 | Progress: 0.83968 +Step 5,150 | Tokens: 168,755,200 | Train Loss EWMA: 6.0969 | Learning Rate: 0.001000 | Progress: 0.84378 +Step 5,175 | Tokens: 169,574,400 | Train Loss EWMA: 6.0983 | Learning Rate: 0.001000 | Progress: 0.84787 +Step 5,200 | Tokens: 170,393,600 | Train Loss EWMA: 6.1003 | Learning Rate: 0.001000 | Progress: 0.85197 +Step 5,225 | Tokens: 171,212,800 | Train Loss EWMA: 6.0972 | Learning Rate: 0.001000 | Progress: 0.85606 +Step 5,250 | Tokens: 172,032,000 | Train Loss EWMA: 6.1020 | Learning Rate: 0.001000 | Progress: 0.86016 +Step 5,275 | Tokens: 172,851,200 | Train Loss EWMA: 6.0998 | Learning Rate: 0.001000 | Progress: 0.86426 +Step 5,300 | Tokens: 173,670,400 | Train Loss EWMA: 6.0966 | Learning Rate: 0.001000 | Progress: 0.86835 +Step 5,325 | Tokens: 174,489,600 | Train Loss EWMA: 6.0962 | Learning Rate: 0.001000 | Progress: 0.87245 +Step 5,350 | Tokens: 175,308,800 | Train Loss EWMA: 6.0924 | Learning Rate: 0.001000 | Progress: 0.87654 +Step 5,375 | Tokens: 176,128,000 | Train Loss EWMA: 6.0934 | Learning Rate: 0.001000 | Progress: 0.88064 +Step 5,400 | Tokens: 176,947,200 | Train Loss EWMA: 6.0838 | Learning Rate: 0.001000 | Progress: 0.88474 +Step 5,425 | Tokens: 177,766,400 | Train Loss EWMA: 6.0831 | Learning Rate: 0.001000 | Progress: 0.88883 +Step 5,450 | Tokens: 178,585,600 | Train Loss EWMA: 6.0790 | Learning Rate: 0.001000 | Progress: 0.89293 +Step 5,475 | Tokens: 179,404,800 | Train Loss EWMA: 6.0824 | Learning Rate: 0.001000 | Progress: 0.89702 +Step 5,500 | Tokens: 180,224,000 | Train Loss EWMA: 6.0789 | Learning Rate: 0.001000 | Progress: 0.90112 +Step 5,525 | Tokens: 181,043,200 | Train Loss EWMA: 6.0786 | Learning Rate: 0.001000 | Progress: 0.90522 +Step 5,550 | Tokens: 181,862,400 | Train Loss EWMA: 6.0758 | Learning Rate: 0.001000 | Progress: 0.90931 +Step 5,575 | Tokens: 182,681,600 | Train Loss EWMA: 6.0792 | Learning Rate: 0.001000 | Progress: 0.91341 +Step 5,600 | Tokens: 183,500,800 | Train Loss EWMA: 6.0788 | Learning Rate: 0.001000 | Progress: 0.91750 +Step 5,625 | Tokens: 184,320,000 | Train Loss EWMA: 6.0724 | Learning Rate: 0.001000 | Progress: 0.92160 +Step 5,650 | Tokens: 185,139,200 | Train Loss EWMA: 6.0687 | Learning Rate: 0.001000 | Progress: 0.92570 +Step 5,675 | Tokens: 185,958,400 | Train Loss EWMA: 6.0649 | Learning Rate: 0.001000 | Progress: 0.92979 +Step 5,700 | Tokens: 186,777,600 | Train Loss EWMA: 6.0637 | Learning Rate: 0.001000 | Progress: 0.93389 +Step 5,725 | Tokens: 187,596,800 | Train Loss EWMA: 6.0595 | Learning Rate: 0.001000 | Progress: 0.93798 +Step 5,750 | Tokens: 188,416,000 | Train Loss EWMA: 6.0649 | Learning Rate: 0.001000 | Progress: 0.94208 +Step 5,775 | Tokens: 189,235,200 | Train Loss EWMA: 6.0561 | Learning Rate: 0.001000 | Progress: 0.94618 +Step 5,800 | Tokens: 190,054,400 | Train Loss EWMA: 6.0584 | Learning Rate: 0.001000 | Progress: 0.95027 +Step 5,825 | Tokens: 190,873,600 | Train Loss EWMA: 6.0622 | Learning Rate: 0.001000 | Progress: 0.95437 +Step 5,850 | Tokens: 191,692,800 | Train Loss EWMA: 6.0657 | Learning Rate: 0.001000 | Progress: 0.95846 +Step 5,875 | Tokens: 192,512,000 | Train Loss EWMA: 6.0626 | Learning Rate: 0.001000 | Progress: 0.96256 +Step 5,900 | Tokens: 193,331,200 | Train Loss EWMA: 6.0602 | Learning Rate: 0.001000 | Progress: 0.96666 +Step 5,925 | Tokens: 194,150,400 | Train Loss EWMA: 6.0630 | Learning Rate: 0.001000 | Progress: 0.97075 +Step 5,950 | Tokens: 194,969,600 | Train Loss EWMA: 6.0621 | Learning Rate: 0.001000 | Progress: 0.97485 +Step 5,975 | Tokens: 195,788,800 | Train Loss EWMA: 6.0626 | Learning Rate: 0.001000 | Progress: 0.97894 +Step 6,000 | Tokens: 196,608,000 | Train Loss EWMA: 6.0558 | Learning Rate: 0.001000 | Progress: 0.98304 +Step 6,025 | Tokens: 197,427,200 | Train Loss EWMA: 6.0611 | Learning Rate: 0.001000 | Progress: 0.98714 +Step 6,050 | Tokens: 198,246,400 | Train Loss EWMA: 6.0535 | Learning Rate: 0.001000 | Progress: 0.99123 +Step 6,075 | Tokens: 199,065,600 | Train Loss EWMA: 6.0536 | Learning Rate: 0.001000 | Progress: 0.99533 +Step 6,100 | Tokens: 199,884,800 | Train Loss EWMA: 6.0489 | Learning Rate: 0.001000 | Progress: 0.99942 diff --git a/wandb/run-20260325_092121-4guua5vm/files/requirements.txt b/wandb/run-20260325_092121-4guua5vm/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..c32285d10ba18c2e783ff2ead305d5976caef668 --- /dev/null +++ b/wandb/run-20260325_092121-4guua5vm/files/requirements.txt @@ -0,0 +1,222 @@ +fsspec==2025.3.0 +PyYAML==6.0.2 +certifi==2025.8.3 +comm==0.2.3 +widgetsnbextension==4.0.14 +Jinja2==3.1.6 +rich==14.1.0 +circuitsvis==1.43.3 +hf-xet==1.1.9 +param==2.2.1 +httpcore==1.0.9 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +asttokens==3.0.0 +filelock==3.19.1 +types-python-dateutil==2.9.0.20250822 +cycler==0.12.1 +stack-data==0.6.3 +jupyter_server==2.17.0 +aiosignal==1.4.0 +xyzservices==2025.4.0 +lark==1.2.2 +ptyprocess==0.7.0 +xxhash==3.5.0 +mpmath==1.3.0 +seaborn==0.13.2 +wadler_lindig==0.1.7 +nbformat==5.10.4 +panel==1.8.0 +accelerate==1.10.1 +plotly==6.3.0 +narwhals==2.4.0 +huggingface-hub==0.34.4 +sentencepiece==0.2.1 +torchvision==0.23.0 +ipython==9.5.0 +tqdm==4.67.1 +contourpy==1.3.3 +nvidia-nvtx-cu12==12.8.90 +nvidia-cuda-runtime-cu12==12.8.90 +yarl==1.20.1 +charset-normalizer==3.4.3 +jupyter-events==0.12.0 +nbclient==0.10.2 +numpy==1.26.4 +decorator==5.2.1 +threadpoolctl==3.6.0 +networkx==3.5 +smmap==5.0.2 +nbconvert==7.16.6 +pytz==2025.2 +aiohappyeyeballs==2.6.1 +requests==2.32.5 +tinycss2==1.4.0 +defusedxml==0.7.1 +matplotlib-inline==0.1.7 +rpds-py==0.27.1 +wandb==0.21.4 +jedi==0.19.2 +pathspec==0.12.1 +transformer-lens==2.16.1 +sympy==1.14.0 +jupyterlab_pygments==0.3.0 +overrides==7.7.0 +notebook_shim==0.2.4 +jupyter==1.1.1 +protobuf==6.32.1 +better-abc==0.0.3 +jsonpointer==3.0.0 +terminado==0.18.1 +cfgv==3.4.0 +rfc3987-syntax==1.1.0 +annotated-types==0.7.0 +pyarrow==21.0.0 +webencodings==0.5.1 +wcwidth==0.2.13 +jupyterlab_server==2.27.3 +argon2-cffi-bindings==25.1.0 +nvidia-nvjitlink-cu12==12.8.93 +jaxtyping==0.3.2 +Pygments==2.19.2 +torch==2.8.0 +rfc3339-validator==0.1.4 +urllib3==2.5.0 +jupyterlab_widgets==3.0.15 +ipykernel==6.30.1 +nvidia-cudnn-cu12==9.10.2.21 +beautifulsoup4==4.13.5 +babel==2.17.0 +pure_eval==0.2.3 +pyparsing==3.2.3 +nvidia-cublas-cu12==12.8.4.1 +regex==2025.9.1 +pycparser==2.23 +soupsieve==2.8 +pytest-cov==7.0.0 +sniffio==1.3.1 +mypy==1.18.1 +notebook==7.4.5 +packaging==25.0 +h11==0.16.0 +psutil==7.0.0 +pexpect==4.9.0 +zstandard==0.25.0 +gitdb==4.0.12 +rfc3986-validator==0.1.1 +pyzmq==27.1.0 +jupyterlab==4.4.7 +toy_models==0.1.0 +torchaudio==2.8.0 +cffi==2.0.0 +mypy_extensions==1.1.0 +attrs==25.3.0 +statsmodels==0.14.6 +transformers==4.56.1 +jupyter_core==5.8.1 +bleach==6.2.0 +fqdn==1.5.1 +async-lru==2.0.5 +nvidia-nccl-cu12==2.27.3 +GitPython==3.1.45 +referencing==0.36.2 +click==8.2.1 +prometheus_client==0.22.1 +bokeh==3.8.0 +httpx==0.28.1 +setuptools==80.9.0 +argon2-cffi==25.1.0 +patsy==1.0.2 +multidict==6.6.4 +pyviz_comms==3.0.6 +arrow==1.3.0 +scikit-learn==1.8.0 +beartype==0.14.1 +ipywidgets==8.1.7 +pydantic_core==2.33.2 +markdown-it-py==4.0.0 +pandas==2.3.2 +virtualenv==20.34.0 +python-dotenv==1.1.1 +isoduration==20.11.0 +python-dateutil==2.9.0.post0 +nodeenv==1.9.1 +nvidia-curand-cu12==10.3.9.90 +webcolors==24.11.1 +MarkupSafe==3.0.2 +nvidia-cusolver-cu12==11.7.3.90 +Send2Trash==1.8.3 +coverage==7.10.6 +jupyter_server_terminals==0.5.3 +debugpy==1.8.16 +json5==0.12.1 +linkify-it-py==2.0.3 +importlib_metadata==8.7.0 +nvidia-cufft-cu12==11.3.3.83 +distlib==0.4.0 +typing-inspection==0.4.1 +identify==2.6.14 +nvidia-cufile-cu12==1.13.1.3 +scipy==1.17.0 +mdurl==0.1.2 +websocket-client==1.8.0 +jsonschema==4.25.1 +python-json-logger==3.3.0 +typing_extensions==4.15.0 +tokenizers==0.22.0 +ipympl==0.9.7 +einops==0.8.1 +jupyter_client==8.6.3 +ipython_pygments_lexers==1.1.1 +h5py==3.14.0 +tabulate==0.9.0 +propcache==0.3.2 +ruff==0.13.0 +tornado==6.5.2 +typeguard==4.4.4 +tomlkit==0.13.2 +pluggy==1.6.0 +pydantic==2.11.7 +zipp==3.23.0 +fancy-einsum==0.0.3 +fastjsonschema==2.21.2 +datasets==4.0.0 +fonttools==4.59.2 +executing==2.2.1 +pillow==11.3.0 +uc-micro-py==1.0.3 +Markdown==3.9 +pre_commit==4.3.0 +aiohttp==3.12.15 +mistune==3.1.4 +tzdata==2025.2 +parso==0.8.5 +triton==3.4.0 +kiwisolver==1.4.9 +idna==3.10 +multiprocess==0.70.16 +dill==0.3.8 +jupyter-lsp==2.3.0 +platformdirs==4.4.0 +sentry-sdk==2.37.1 +prompt_toolkit==3.0.52 +jsonschema-specifications==2025.9.1 +pytest==8.4.2 +mdit-py-plugins==0.5.0 +transformers-stream-generator==0.0.5 +nvidia-cusparselt-cu12==0.7.1 +joblib==1.5.3 +pandocfilters==1.5.1 +jupyter-console==6.6.3 +anyio==4.10.0 +six==1.17.0 +holoviews==1.21.0 +matplotlib==3.10.6 +colorcet==3.1.0 +uri-template==1.3.0 +nest-asyncio==1.6.0 +nvidia-cusparse-cu12==12.5.8.93 +iniconfig==2.1.0 +traitlets==5.14.3 +safetensors==0.6.2 +frozenlist==1.7.0 diff --git a/wandb/run-20260325_092121-4guua5vm/files/wandb-metadata.json b/wandb/run-20260325_092121-4guua5vm/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..9bd0dfb0048ea337e7e146d4a3aa7e7db0f8bc99 --- /dev/null +++ b/wandb/run-20260325_092121-4guua5vm/files/wandb-metadata.json @@ -0,0 +1,38 @@ +{ + "os": "Linux-5.19.0-45-generic-x86_64-with-glibc2.35", + "python": "CPython 3.11.7", + "startedAt": "2026-03-25T09:21:21.907528Z", + "program": "", + "git": { + "remote": "https://github.com/jgroh3/toy_models.git", + "commit": "6d40e57307f8f7c2c410c6d3dac053261e8a836d" + }, + "email": "tzfof8@gmail.com", + "root": "/notebooks/toy_models/model_training/model", + "host": "nwlkget26z", + "executable": "/notebooks/toy_models/.toy_models_env/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 1, + "disk": { + "/": { + "total": "262240792576", + "used": "121270431744" + } + }, + "memory": { + "total": "47332843520" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere", + "uuid": "GPU-dabc88f0-4b6a-64c4-f799-f3f38a477a96" + } + ], + "cudaVersion": "12.4", + "writerId": "sflf975jycouqau4pywajxey1ffzx7ln" +} \ No newline at end of file diff --git a/wandb/run-20260325_092121-4guua5vm/files/wandb-summary.json b/wandb/run-20260325_092121-4guua5vm/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..a5b35baaed25eb3d4bf1bc3b4b4b1cddadf71b5f --- /dev/null +++ b/wandb/run-20260325_092121-4guua5vm/files/wandb-summary.json @@ -0,0 +1 @@ +{"tokens_seen":199884800,"train_loss_ewma":6.0489037641190455,"train_loss":5.924057483673096,"_step":6100,"_wandb":{"runtime":3219},"step":6100,"tokens_per_second":32768,"progress":0.999424,"learning_rate":0.001,"_timestamp":1.7744336999308636e+09,"_runtime":3219.170095227} \ No newline at end of file diff --git a/wandb/run-20260325_092121-4guua5vm/logs/debug-core.log b/wandb/run-20260325_092121-4guua5vm/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..4072f3213d5dad001210cf98f8a22b2011812a8e --- /dev/null +++ b/wandb/run-20260325_092121-4guua5vm/logs/debug-core.log @@ -0,0 +1,16 @@ +{"time":"2026-03-25T09:21:22.295279152Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp287p4est/port-3604.txt","pid":3604,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-03-25T09:21:22.297564471Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":3604} +{"time":"2026-03-25T09:21:22.297539717Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3604-3662-227623527/socket","Net":"unix"}} +{"time":"2026-03-25T09:21:22.394822808Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-03-25T09:21:22.413922307Z","level":"INFO","msg":"handleInformInit: received","streamId":"4guua5vm","id":"1(@)"} +{"time":"2026-03-25T09:21:22.675413461Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"4guua5vm","id":"1(@)"} +{"time":"2026-03-25T10:15:12.704908196Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"4guua5vm","id":"1(@)"} +{"time":"2026-03-25T10:15:12.708685098Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"4guua5vm","id":"1(@)"} +{"time":"2026-03-25T10:15:12.708725643Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-03-25T10:15:12.708746636Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-03-25T10:15:12.708760385Z","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-03-25T10:15:12.708805828Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-03-25T10:15:12.708810429Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-03-25T10:15:12.708831062Z","level":"INFO","msg":"server is shutting down"} +{"time":"2026-03-25T10:15:12.708985835Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3604-3662-227623527/socket","Net":"unix"}} +{"time":"2026-03-25T10:15:12.709025794Z","level":"INFO","msg":"server is closed"} diff --git a/wandb/run-20260325_092121-4guua5vm/logs/debug-internal.log b/wandb/run-20260325_092121-4guua5vm/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..72c863f7cb04979be388bc3b7d0e5baae06628d6 --- /dev/null +++ b/wandb/run-20260325_092121-4guua5vm/logs/debug-internal.log @@ -0,0 +1,12 @@ +{"time":"2026-03-25T09:21:22.41420794Z","level":"INFO","msg":"stream: starting","core version":"0.21.4"} +{"time":"2026-03-25T09:21:22.675339572Z","level":"INFO","msg":"stream: created new stream","id":"4guua5vm"} +{"time":"2026-03-25T09:21:22.675405951Z","level":"INFO","msg":"stream: started","id":"4guua5vm"} +{"time":"2026-03-25T09:21:22.675456409Z","level":"INFO","msg":"sender: started","stream_id":"4guua5vm"} +{"time":"2026-03-25T09:21:22.675452087Z","level":"INFO","msg":"writer: started","stream_id":"4guua5vm"} +{"time":"2026-03-25T09:21:22.675491769Z","level":"INFO","msg":"handler: started","stream_id":"4guua5vm"} +{"time":"2026-03-25T10:15:02.459446173Z","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"uploading history steps 241-241, summary, console lines 250-250","runtime_seconds":9.164065763},{"desc":"updating run metadata","runtime_seconds":0.001753079}],"total_operations":2}} +{"time":"2026-03-25T10:15:02.93303388Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-03-25T10:15:12.704963398Z","level":"INFO","msg":"stream: closing","id":"4guua5vm"} +{"time":"2026-03-25T10:15:12.705022861Z","level":"INFO","msg":"handler: closed","stream_id":"4guua5vm"} +{"time":"2026-03-25T10:15:12.705073682Z","level":"INFO","msg":"sender: closed","stream_id":"4guua5vm"} +{"time":"2026-03-25T10:15:12.705122378Z","level":"INFO","msg":"stream: closed","id":"4guua5vm"} diff --git a/wandb/run-20260325_092121-4guua5vm/logs/debug.log b/wandb/run-20260325_092121-4guua5vm/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..6ec9be4241b6530990520ecbefee02d9f563309c --- /dev/null +++ b/wandb/run-20260325_092121-4guua5vm/logs/debug.log @@ -0,0 +1,26 @@ +2026-03-25 09:21:21,913 INFO MainThread:3604 [wandb_setup.py:_flush():81] Current SDK version is 0.21.4 +2026-03-25 09:21:21,913 INFO MainThread:3604 [wandb_setup.py:_flush():81] Configure stats pid to 3604 +2026-03-25 09:21:21,913 INFO MainThread:3604 [wandb_setup.py:_flush():81] Loading settings from /root/.config/wandb/settings +2026-03-25 09:21:21,913 INFO MainThread:3604 [wandb_setup.py:_flush():81] Loading settings from /notebooks/toy_models/model_training/model/wandb/settings +2026-03-25 09:21:21,913 INFO MainThread:3604 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-03-25 09:21:21,913 INFO MainThread:3604 [wandb_init.py:setup_run_log_directory():686] Logging user logs to /notebooks/toy_models/model_training/model/wandb/run-20260325_092121-4guua5vm/logs/debug.log +2026-03-25 09:21:21,913 INFO MainThread:3604 [wandb_init.py:setup_run_log_directory():687] Logging internal logs to /notebooks/toy_models/model_training/model/wandb/run-20260325_092121-4guua5vm/logs/debug-internal.log +2026-03-25 09:21:21,913 INFO MainThread:3604 [wandb_init.py:init():813] calling init triggers +2026-03-25 09:21:21,913 INFO MainThread:3604 [wandb_init.py:init():818] wandb.init called with sweep_config: {} +config: {'model_name': 'pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2', 'n_layers': 2, 'd_model': 512, 'd_mlp': 2048, 'd_head': 64, 'n_heads': 8, 'attn_only': False, 'layer_norm_eps': 1e-05, 'init_range': 0.02, 'n_ctx': 1024, 'd_vocab': 32000, 'dataset_name': 'eoinf/pile_llama_mix_within_rows_pile_all_random_tokens_uniform_frac0d2', 'tokenizer_name': '', 'seed': 10, 'data_seed': 10, 'device': 'cuda', 'use_bfloat16_matmul': False, 'batch_size_per_device': 32, 'n_devices': 1, 'batches_per_step': 1, 'max_tokens': 200000000, 'lr_hidden': 0.001, 'lr_vector': 0.0005, 'lr_schedule': 'constant_with_warmup', 'warmup_tokens': 30000000, 'weight_decay': 0.05, 'grad_norm_clip': 1.0, 'train_loss_moving_average_beta': 0.99, 'log_interval': 25, 'save_checkpoints': True, 'checkpoint_interval': 500, 'checkpoint_interval_ratio': 1.1, 'save_log_checkpoints': True, 'use_wandb': True, 'batch_size': 32, 'tokens_per_step': 32768, 'warmup_steps': 915, 'max_steps': 6103, '_wandb': {}} +2026-03-25 09:21:21,913 INFO MainThread:3604 [wandb_init.py:init():854] starting backend +2026-03-25 09:21:22,395 INFO MainThread:3604 [wandb_init.py:init():857] sending inform_init request +2026-03-25 09:21:22,409 INFO MainThread:3604 [wandb_init.py:init():865] backend started and connected +2026-03-25 09:21:22,411 INFO MainThread:3604 [wandb_init.py:init():936] updated telemetry +2026-03-25 09:21:22,417 INFO MainThread:3604 [wandb_init.py:init():960] communicating run to backend with 90.0 second timeout +2026-03-25 09:21:23,291 INFO MainThread:3604 [wandb_init.py:init():1011] starting run threads in backend +2026-03-25 09:21:24,185 INFO MainThread:3604 [wandb_run.py:_console_start():2506] atexit reg +2026-03-25 09:21:24,186 INFO MainThread:3604 [wandb_run.py:_redirect():2354] redirect: wrap_raw +2026-03-25 09:21:24,186 INFO MainThread:3604 [wandb_run.py:_redirect():2423] Wrapping output streams. +2026-03-25 09:21:24,186 INFO MainThread:3604 [wandb_run.py:_redirect():2446] Redirects installed. +2026-03-25 09:21:24,197 INFO MainThread:3604 [wandb_init.py:init():1049] run started, returning control to user process +2026-03-25 10:15:02,444 INFO MainThread:3604 [wandb_run.py:_finish():2272] finishing run tzach/toy-transformer-replication/4guua5vm +2026-03-25 10:15:02,456 INFO MainThread:3604 [wandb_run.py:_atexit_cleanup():2471] got exitcode: 0 +2026-03-25 10:15:02,457 INFO MainThread:3604 [wandb_run.py:_restore():2453] restore +2026-03-25 10:15:02,457 INFO MainThread:3604 [wandb_run.py:_restore():2459] restore done +2026-03-25 10:15:12,704 INFO MainThread:3604 [wandb_run.py:_footer_sync_info():3867] logging synced files diff --git a/wandb/run-20260325_092121-4guua5vm/run-4guua5vm.wandb b/wandb/run-20260325_092121-4guua5vm/run-4guua5vm.wandb new file mode 100644 index 0000000000000000000000000000000000000000..b14a86d71585aa1567446f343fe7263367ad2519 --- /dev/null +++ b/wandb/run-20260325_092121-4guua5vm/run-4guua5vm.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a1f36d4eb3243350e3fdd24db46a1f75e83bdd8c7454191538ddaa56651f948 +size 3539547