| { | |
| "dataset_name": "c4", | |
| "max_seq_len": 256, | |
| "enc_dec_model": "facebook/bart-base", | |
| "train_batch_size": 32, | |
| "eval_batch_size": 16, | |
| "num_encoder_latents": 128, | |
| "num_decoder_latents": 128, | |
| "dim_ae": 768, | |
| "num_layers": 3, | |
| "l2_normalize_latents": true, | |
| "output_dir": "saved_latent_models/c4/2024-09-22_22-29-54", | |
| "save_dir": "saved_latent_models", | |
| "learning_rate": 0.0001, | |
| "num_train_steps": 50000, | |
| "lr_schedule": "linear", | |
| "lr_warmup_steps": 1000, | |
| "optimizer": "adamw", | |
| "adam_beta1": 0.9, | |
| "adam_beta2": 0.999, | |
| "adam_weight_decay": 0.01, | |
| "eval_every": 1000, | |
| "mixed_precision": "no", | |
| "wandb_name": "bart-c4-l2norm-128-768", | |
| "lm_mode": "freeze", | |
| "eval": false, | |
| "resume_training": false, | |
| "resume_dir": null, | |
| "num_devices": 1 | |
| } |