Upload folder using huggingface_hub

len2_5120/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

len2_5120/04.pretrain.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+set -e
+
+
+# --- initialize conda ---
+source /root/miniconda3/etc/profile.d/conda.sh
+
+# --- activate env ---
+conda activate bpe_v2
+
+work_dir=/root/NaN/dna-tokenizer
+data_dir=${work_dir}/dna-tokenizer-data
+cache_dir=${data_dir}/cache
+
+
+# runnum=len2_2048
+runnum=len2_5120
+
+mkdir -p ${work_dir}/pretrain/models/model_${runnum}
+output_dir=${work_dir}/pretrain/models/model_${runnum}
+
+
+cp 04.pretrain.sh ${output_dir}
+torchrun --nproc_per_node=8  run_mlm.py \
+					--output_dir ${output_dir} \
+					--model_type bert \
+					--tokenizer_name ${work_dir}/merge_bpe/vocab_5120/merge_tokenizer_unigram_len2.json \
+					--config_name ${work_dir}/BPE_merge/config_5120.json \
+					--project_name token_eval_len2_5120 \
+					--do_train True \
+					--model_max_length 512 \
+					--max_seq_length 512 \
+					--line_by_line True \
+					--pad_to_max_length True \
+					--train_file ${data_dir}/merge_bpe_5120_allchr_all_tokenized_train_chrOnly.tsv \
+					--validation_file ${data_dir}/merge_bpe_5120_allchr_all_tokenized_val_chrOnly.tsv \
+					--cache_dir ${cache_dir} \
+					--use_fast_tokenizer True \
+					--do_eval True \
+					--gradient_accumulation_steps 1 \
+					--per_device_train_batch_size 96 \
+					--per_device_eval_batch_size 96 \
+					--save_steps 1000 \
+					--save_total_limit 10 \
+					--max_steps 100000 \
+					--logging_steps 1000 \
+					--learning_rate 4e-5 \
+					--adam_epsilon 1e-6 \
+					--weight_decay 0.01 \
+					--adam_beta1 0.9 \
+					--adam_beta2 0.98 \
+					--mlm_probability 0.15 \
+					--warmup_steps 10000 \
+					--seed 42 \
+					--preprocessing_num_workers 8 \
+					--overwrite_output_dir True
+					#--resume_from_checkpoint /root/NaN/dna-tokenizer/pretrain/models/model_len2_4096/checkpoint-36000
+
+
+
+
+
+
+
+
+
+

len2_5120/04.pretrain_200k.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+# Activate conda
+source ~/miniconda3/etc/profile.d/conda.sh
+conda activate bpe
+
+# select GPU
+export CUDA_VISIBLE_DEVICES=6
+
+work_dir=/home/n5huang/dna_token
+data_dir=${work_dir}/output_tokens
+cache_dir=${data_dir}/cache
+
+runnum=len2_5120
+
+# -p to use existing directory
+mkdir -p ${work_dir}/pretrain/models/${runnum}
+output_dir=${work_dir}/pretrain/models/${runnum}
+
+# CHANGED: 
+# --config_name ${work_dir}/BPE_merge/config_5120.json -> --config_name ${work_dir}/pretrain/models/len2_5120/config.json 
+# --tokenizer_name ${work_dir}/merge_bpe/... -> --tokenizer_name ${work_dir}/tokenizer_evaluation/merge_bpe/...
+# all tf-idf -> len2_5120 (model, tokenizer, data)
+# ADDED:
+# --resume_from_checkpoint /home/n5huang/dna_token/pretrain/models/len2_5120/checkpoint-100000
+# REMOVED:
+# overwrite_output_dir True
+
+cp /home/n5huang/dna-tokenizer/BPE_merge/04.pretrain_200k.sh ${output_dir}
+python /home/n5huang/dna-tokenizer/BPE_merge/run_mlm_200k.py \
+					--output_dir ${output_dir} \
+					--model_type bert \
+					--tokenizer_name ${work_dir}/tokenizer_evaluation/merge_bpe/vocab_5120/merge_tokenizer_unigram_len2.json \
+					--config_name ${work_dir}/pretrain/models/len2_5120/config.json \
+					--project_name token_eval_len2_5120 \
+					--do_train True \
+					--model_max_length 512 \
+					--max_seq_length 512 \
+					--line_by_line True \
+					--pad_to_max_length True \
+					--train_file ${data_dir}/merge_bpe_5120_allchr_all_tokenized_train_chrOnly.tsv \
+					--validation_file ${data_dir}/merge_bpe_5120_allchr_all_tokenized_val_chrOnly.tsv \
+					--cache_dir ${cache_dir} \
+					--use_fast_tokenizer True \
+					--do_eval True \
+					--gradient_accumulation_steps 1 \
+					--per_device_train_batch_size 96 \
+					--per_device_eval_batch_size 96 \
+					--save_steps 1000 \
+					--save_total_limit 10 \
+					--max_steps 200000 \
+					--logging_steps 1000 \
+					--learning_rate 4e-5 \
+					--adam_epsilon 1e-6 \
+					--weight_decay 0.01 \
+					--adam_beta1 0.9 \
+					--adam_beta2 0.98 \
+					--mlm_probability 0.15 \
+					--warmup_steps 10000 \
+					--seed 42 \
+					--preprocessing_num_workers 8 \
+					--resume_from_checkpoint /home/n5huang/dna_token/pretrain/models/len2_5120/checkpoint-100000
+

len2_5120/README.md

@@ -0,0 +1,56 @@
+---
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: len2_5120
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# len2_5120
+
+This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 4.9009
+- Accuracy: 0.3043
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 4e-05
+- train_batch_size: 96
+- eval_batch_size: 96
+- seed: 42
+- optimizer: Adam with betas=(0.9,0.98) and epsilon=1e-06
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 10000
+- training_steps: 200000
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.35.2
+- Pytorch 2.8.0+cu128
+- Datasets 4.4.1
+- Tokenizers 0.15.2

len2_5120/all_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 19.54,
+    "eval_accuracy": 0.30433709022367145,
+    "eval_loss": 4.900882244110107,
+    "eval_runtime": 1259.8218,
+    "eval_samples": 109173,
+    "eval_samples_per_second": 86.657,
+    "eval_step": 200000,
+    "eval_steps_per_second": 0.903,
+    "perplexity": 134.40830833002153,
+    "train_loss": 2.48521353515625,
+    "train_runtime": 258545.0504,
+    "train_samples": 982503,
+    "train_samples_per_second": 74.262,
+    "train_steps_per_second": 0.774
+}

len2_5120/checkpoint-191000/config.json

@@ -0,0 +1,25 @@
+{
+  "architectures": [
+    "BertForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_length": 512,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.35.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 5120
+}

len2_5120/checkpoint-191000/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "max_length": 512,
+  "pad_token_id": 0,
+  "transformers_version": "4.35.2"
+}

len2_5120/checkpoint-191000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3d088c80a86c3b8ceec56a90523c71b08b41b05395390ca7974259e7b86b46d
+size 359944256

len2_5120/checkpoint-191000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce8464e2e6a6ffe607fd3443edb93081a68f33e5002d9a3412e9dd405ea3e667
+size 720010699

len2_5120/checkpoint-191000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74d64fa3c1245c787a712ad0a6d7fd62111d9933515a68c0b6ede8fcd6d12bbd
+size 14645

len2_5120/checkpoint-191000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03801543841a49721ffa8d88e86d4edc246ccb0f1b28ddcb8c3e3197118fe676
+size 1465

len2_5120/checkpoint-191000/trainer_state.json

@@ -0,0 +1,1165 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 18.661455788959454,
+  "eval_steps": 500,
+  "global_step": 191000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.78,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 7.5832,
+      "step": 1000
+    },
+    {
+      "epoch": 1.56,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 6.9724,
+      "step": 2000
+    },
+    {
+      "epoch": 2.34,
+      "learning_rate": 1.2e-05,
+      "loss": 6.7695,
+      "step": 3000
+    },
+    {
+      "epoch": 3.12,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 6.7002,
+      "step": 4000
+    },
+    {
+      "epoch": 3.91,
+      "learning_rate": 2e-05,
+      "loss": 6.666,
+      "step": 5000
+    },
+    {
+      "epoch": 4.69,
+      "learning_rate": 2.4e-05,
+      "loss": 6.6503,
+      "step": 6000
+    },
+    {
+      "epoch": 5.47,
+      "learning_rate": 2.8e-05,
+      "loss": 6.6407,
+      "step": 7000
+    },
+    {
+      "epoch": 6.25,
+      "learning_rate": 3.2000000000000005e-05,
+      "loss": 6.6277,
+      "step": 8000
+    },
+    {
+      "epoch": 7.03,
+      "learning_rate": 3.6e-05,
+      "loss": 6.6209,
+      "step": 9000
+    },
+    {
+      "epoch": 7.81,
+      "learning_rate": 4e-05,
+      "loss": 6.6135,
+      "step": 10000
+    },
+    {
+      "epoch": 8.59,
+      "learning_rate": 3.955555555555556e-05,
+      "loss": 6.6084,
+      "step": 11000
+    },
+    {
+      "epoch": 9.38,
+      "learning_rate": 3.9111111111111115e-05,
+      "loss": 6.6041,
+      "step": 12000
+    },
+    {
+      "epoch": 10.16,
+      "learning_rate": 3.866666666666667e-05,
+      "loss": 6.6013,
+      "step": 13000
+    },
+    {
+      "epoch": 10.94,
+      "learning_rate": 3.8222222222222226e-05,
+      "loss": 6.5994,
+      "step": 14000
+    },
+    {
+      "epoch": 11.72,
+      "learning_rate": 3.777777777777778e-05,
+      "loss": 6.594,
+      "step": 15000
+    },
+    {
+      "epoch": 12.5,
+      "learning_rate": 3.733333333333334e-05,
+      "loss": 6.5925,
+      "step": 16000
+    },
+    {
+      "epoch": 13.28,
+      "learning_rate": 3.6888888888888896e-05,
+      "loss": 6.5893,
+      "step": 17000
+    },
+    {
+      "epoch": 14.06,
+      "learning_rate": 3.644444444444445e-05,
+      "loss": 6.5875,
+      "step": 18000
+    },
+    {
+      "epoch": 14.84,
+      "learning_rate": 3.6e-05,
+      "loss": 6.5232,
+      "step": 19000
+    },
+    {
+      "epoch": 15.62,
+      "learning_rate": 3.555555555555555e-05,
+      "loss": 6.4358,
+      "step": 20000
+    },
+    {
+      "epoch": 16.41,
+      "learning_rate": 3.511111111111111e-05,
+      "loss": 6.3712,
+      "step": 21000
+    },
+    {
+      "epoch": 17.19,
+      "learning_rate": 3.466666666666667e-05,
+      "loss": 6.311,
+      "step": 22000
+    },
+    {
+      "epoch": 17.97,
+      "learning_rate": 3.4222222222222224e-05,
+      "loss": 6.2493,
+      "step": 23000
+    },
+    {
+      "epoch": 18.75,
+      "learning_rate": 3.377777777777778e-05,
+      "loss": 6.1597,
+      "step": 24000
+    },
+    {
+      "epoch": 19.53,
+      "learning_rate": 3.3333333333333335e-05,
+      "loss": 6.0276,
+      "step": 25000
+    },
+    {
+      "epoch": 20.31,
+      "learning_rate": 3.288888888888889e-05,
+      "loss": 5.8317,
+      "step": 26000
+    },
+    {
+      "epoch": 21.09,
+      "learning_rate": 3.2444444444444446e-05,
+      "loss": 5.6782,
+      "step": 27000
+    },
+    {
+      "epoch": 21.88,
+      "learning_rate": 3.2000000000000005e-05,
+      "loss": 5.5975,
+      "step": 28000
+    },
+    {
+      "epoch": 22.66,
+      "learning_rate": 3.155555555555556e-05,
+      "loss": 5.5369,
+      "step": 29000
+    },
+    {
+      "epoch": 23.44,
+      "learning_rate": 3.111111111111112e-05,
+      "loss": 5.4921,
+      "step": 30000
+    },
+    {
+      "epoch": 24.22,
+      "learning_rate": 3.066666666666667e-05,
+      "loss": 5.4554,
+      "step": 31000
+    },
+    {
+      "epoch": 25.0,
+      "learning_rate": 3.0222222222222225e-05,
+      "loss": 5.4237,
+      "step": 32000
+    },
+    {
+      "epoch": 25.78,
+      "learning_rate": 2.977777777777778e-05,
+      "loss": 5.3938,
+      "step": 33000
+    },
+    {
+      "epoch": 26.56,
+      "learning_rate": 2.9333333333333333e-05,
+      "loss": 5.3701,
+      "step": 34000
+    },
+    {
+      "epoch": 27.34,
+      "learning_rate": 2.888888888888889e-05,
+      "loss": 5.3452,
+      "step": 35000
+    },
+    {
+      "epoch": 28.12,
+      "learning_rate": 2.8444444444444447e-05,
+      "loss": 5.327,
+      "step": 36000
+    },
+    {
+      "epoch": 28.91,
+      "learning_rate": 2.8e-05,
+      "loss": 5.3063,
+      "step": 37000
+    },
+    {
+      "epoch": 29.69,
+      "learning_rate": 2.755555555555556e-05,
+      "loss": 5.2905,
+      "step": 38000
+    },
+    {
+      "epoch": 30.47,
+      "learning_rate": 2.7111111111111114e-05,
+      "loss": 5.2747,
+      "step": 39000
+    },
+    {
+      "epoch": 31.25,
+      "learning_rate": 2.6666666666666667e-05,
+      "loss": 5.2596,
+      "step": 40000
+    },
+    {
+      "epoch": 32.03,
+      "learning_rate": 2.6222222222222226e-05,
+      "loss": 5.2483,
+      "step": 41000
+    },
+    {
+      "epoch": 32.81,
+      "learning_rate": 2.577777777777778e-05,
+      "loss": 5.2336,
+      "step": 42000
+    },
+    {
+      "epoch": 33.59,
+      "learning_rate": 2.5333333333333334e-05,
+      "loss": 5.2214,
+      "step": 43000
+    },
+    {
+      "epoch": 34.38,
+      "learning_rate": 2.4888888888888893e-05,
+      "loss": 5.2103,
+      "step": 44000
+    },
+    {
+      "epoch": 35.16,
+      "learning_rate": 2.444444444444445e-05,
+      "loss": 5.2015,
+      "step": 45000
+    },
+    {
+      "epoch": 35.94,
+      "learning_rate": 2.4e-05,
+      "loss": 5.1907,
+      "step": 46000
+    },
+    {
+      "epoch": 36.72,
+      "learning_rate": 2.355555555555556e-05,
+      "loss": 5.1801,
+      "step": 47000
+    },
+    {
+      "epoch": 37.5,
+      "learning_rate": 2.3111111111111112e-05,
+      "loss": 5.1713,
+      "step": 48000
+    },
+    {
+      "epoch": 38.28,
+      "learning_rate": 2.2666666666666668e-05,
+      "loss": 5.1645,
+      "step": 49000
+    },
+    {
+      "epoch": 39.06,
+      "learning_rate": 2.2222222222222227e-05,
+      "loss": 5.1548,
+      "step": 50000
+    },
+    {
+      "epoch": 39.84,
+      "learning_rate": 2.177777777777778e-05,
+      "loss": 5.1462,
+      "step": 51000
+    },
+    {
+      "epoch": 40.62,
+      "learning_rate": 2.1333333333333335e-05,
+      "loss": 5.1417,
+      "step": 52000
+    },
+    {
+      "epoch": 41.41,
+      "learning_rate": 2.088888888888889e-05,
+      "loss": 5.1305,
+      "step": 53000
+    },
+    {
+      "epoch": 42.19,
+      "learning_rate": 2.0444444444444446e-05,
+      "loss": 5.1287,
+      "step": 54000
+    },
+    {
+      "epoch": 42.97,
+      "learning_rate": 2e-05,
+      "loss": 5.1202,
+      "step": 55000
+    },
+    {
+      "epoch": 43.75,
+      "learning_rate": 1.9555555555555557e-05,
+      "loss": 5.1139,
+      "step": 56000
+    },
+    {
+      "epoch": 44.53,
+      "learning_rate": 1.9111111111111113e-05,
+      "loss": 5.1081,
+      "step": 57000
+    },
+    {
+      "epoch": 45.31,
+      "learning_rate": 1.866666666666667e-05,
+      "loss": 5.1032,
+      "step": 58000
+    },
+    {
+      "epoch": 46.09,
+      "learning_rate": 1.8222222222222224e-05,
+      "loss": 5.0968,
+      "step": 59000
+    },
+    {
+      "epoch": 46.88,
+      "learning_rate": 1.7777777777777777e-05,
+      "loss": 5.0914,
+      "step": 60000
+    },
+    {
+      "epoch": 47.66,
+      "learning_rate": 1.7333333333333336e-05,
+      "loss": 5.085,
+      "step": 61000
+    },
+    {
+      "epoch": 48.44,
+      "learning_rate": 1.688888888888889e-05,
+      "loss": 5.0806,
+      "step": 62000
+    },
+    {
+      "epoch": 49.22,
+      "learning_rate": 1.6444444444444444e-05,
+      "loss": 5.0769,
+      "step": 63000
+    },
+    {
+      "epoch": 50.0,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 5.0728,
+      "step": 64000
+    },
+    {
+      "epoch": 50.78,
+      "learning_rate": 1.555555555555556e-05,
+      "loss": 5.0692,
+      "step": 65000
+    },
+    {
+      "epoch": 51.56,
+      "learning_rate": 1.5111111111111112e-05,
+      "loss": 5.065,
+      "step": 66000
+    },
+    {
+      "epoch": 52.34,
+      "learning_rate": 1.4666666666666666e-05,
+      "loss": 5.0594,
+      "step": 67000
+    },
+    {
+      "epoch": 53.12,
+      "learning_rate": 1.4222222222222224e-05,
+      "loss": 5.0549,
+      "step": 68000
+    },
+    {
+      "epoch": 53.91,
+      "learning_rate": 1.377777777777778e-05,
+      "loss": 5.0538,
+      "step": 69000
+    },
+    {
+      "epoch": 54.69,
+      "learning_rate": 1.3333333333333333e-05,
+      "loss": 5.0482,
+      "step": 70000
+    },
+    {
+      "epoch": 55.47,
+      "learning_rate": 1.288888888888889e-05,
+      "loss": 5.0475,
+      "step": 71000
+    },
+    {
+      "epoch": 56.25,
+      "learning_rate": 1.2444444444444446e-05,
+      "loss": 5.0399,
+      "step": 72000
+    },
+    {
+      "epoch": 57.03,
+      "learning_rate": 1.2e-05,
+      "loss": 5.0406,
+      "step": 73000
+    },
+    {
+      "epoch": 57.81,
+      "learning_rate": 1.1555555555555556e-05,
+      "loss": 5.037,
+      "step": 74000
+    },
+    {
+      "epoch": 58.59,
+      "learning_rate": 1.1111111111111113e-05,
+      "loss": 5.0326,
+      "step": 75000
+    },
+    {
+      "epoch": 59.38,
+      "learning_rate": 1.0666666666666667e-05,
+      "loss": 5.031,
+      "step": 76000
+    },
+    {
+      "epoch": 60.16,
+      "learning_rate": 1.0222222222222223e-05,
+      "loss": 5.0291,
+      "step": 77000
+    },
+    {
+      "epoch": 60.94,
+      "learning_rate": 9.777777777777779e-06,
+      "loss": 5.0266,
+      "step": 78000
+    },
+    {
+      "epoch": 61.72,
+      "learning_rate": 9.333333333333334e-06,
+      "loss": 5.0235,
+      "step": 79000
+    },
+    {
+      "epoch": 62.5,
+      "learning_rate": 8.888888888888888e-06,
+      "loss": 5.0211,
+      "step": 80000
+    },
+    {
+      "epoch": 63.28,
+      "learning_rate": 8.444444444444446e-06,
+      "loss": 5.0214,
+      "step": 81000
+    },
+    {
+      "epoch": 64.06,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 5.0176,
+      "step": 82000
+    },
+    {
+      "epoch": 64.84,
+      "learning_rate": 7.555555555555556e-06,
+      "loss": 5.0157,
+      "step": 83000
+    },
+    {
+      "epoch": 65.62,
+      "learning_rate": 7.111111111111112e-06,
+      "loss": 5.014,
+      "step": 84000
+    },
+    {
+      "epoch": 66.41,
+      "learning_rate": 6.666666666666667e-06,
+      "loss": 5.0115,
+      "step": 85000
+    },
+    {
+      "epoch": 67.19,
+      "learning_rate": 6.222222222222223e-06,
+      "loss": 5.0087,
+      "step": 86000
+    },
+    {
+      "epoch": 67.97,
+      "learning_rate": 5.777777777777778e-06,
+      "loss": 5.0105,
+      "step": 87000
+    },
+    {
+      "epoch": 68.75,
+      "learning_rate": 5.333333333333334e-06,
+      "loss": 5.0062,
+      "step": 88000
+    },
+    {
+      "epoch": 69.53,
+      "learning_rate": 4.888888888888889e-06,
+      "loss": 5.0028,
+      "step": 89000
+    },
+    {
+      "epoch": 70.31,
+      "learning_rate": 4.444444444444444e-06,
+      "loss": 5.0066,
+      "step": 90000
+    },
+    {
+      "epoch": 71.09,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 5.0032,
+      "step": 91000
+    },
+    {
+      "epoch": 71.88,
+      "learning_rate": 3.555555555555556e-06,
+      "loss": 5.0022,
+      "step": 92000
+    },
+    {
+      "epoch": 72.66,
+      "learning_rate": 3.1111111111111116e-06,
+      "loss": 5.0024,
+      "step": 93000
+    },
+    {
+      "epoch": 73.44,
+      "learning_rate": 2.666666666666667e-06,
+      "loss": 5.0007,
+      "step": 94000
+    },
+    {
+      "epoch": 74.22,
+      "learning_rate": 2.222222222222222e-06,
+      "loss": 5.0008,
+      "step": 95000
+    },
+    {
+      "epoch": 75.0,
+      "learning_rate": 1.777777777777778e-06,
+      "loss": 4.9978,
+      "step": 96000
+    },
+    {
+      "epoch": 75.78,
+      "learning_rate": 1.3333333333333334e-06,
+      "loss": 4.9966,
+      "step": 97000
+    },
+    {
+      "epoch": 76.56,
+      "learning_rate": 8.88888888888889e-07,
+      "loss": 4.9994,
+      "step": 98000
+    },
+    {
+      "epoch": 77.34,
+      "learning_rate": 4.444444444444445e-07,
+      "loss": 4.9966,
+      "step": 99000
+    },
+    {
+      "epoch": 78.12,
+      "learning_rate": 0.0,
+      "loss": 4.9976,
+      "step": 100000
+    },
+    {
+      "epoch": 9.87,
+      "learning_rate": 2.0842105263157895e-05,
+      "loss": 5.0155,
+      "step": 101000
+    },
+    {
+      "epoch": 9.97,
+      "learning_rate": 2.0631578947368425e-05,
+      "loss": 5.024,
+      "step": 102000
+    },
+    {
+      "epoch": 10.06,
+      "learning_rate": 2.0421052631578952e-05,
+      "loss": 5.0241,
+      "step": 103000
+    },
+    {
+      "epoch": 10.16,
+      "learning_rate": 2.0210526315789475e-05,
+      "loss": 5.0218,
+      "step": 104000
+    },
+    {
+      "epoch": 10.26,
+      "learning_rate": 2e-05,
+      "loss": 5.0231,
+      "step": 105000
+    },
+    {
+      "epoch": 10.36,
+      "learning_rate": 1.9789473684210528e-05,
+      "loss": 5.0231,
+      "step": 106000
+    },
+    {
+      "epoch": 10.45,
+      "learning_rate": 1.9578947368421055e-05,
+      "loss": 5.0198,
+      "step": 107000
+    },
+    {
+      "epoch": 10.55,
+      "learning_rate": 1.936842105263158e-05,
+      "loss": 5.0158,
+      "step": 108000
+    },
+    {
+      "epoch": 10.65,
+      "learning_rate": 1.9157894736842108e-05,
+      "loss": 5.0122,
+      "step": 109000
+    },
+    {
+      "epoch": 10.75,
+      "learning_rate": 1.894736842105263e-05,
+      "loss": 5.0165,
+      "step": 110000
+    },
+    {
+      "epoch": 10.85,
+      "learning_rate": 1.873684210526316e-05,
+      "loss": 5.0183,
+      "step": 111000
+    },
+    {
+      "epoch": 10.94,
+      "learning_rate": 1.8526315789473684e-05,
+      "loss": 5.01,
+      "step": 112000
+    },
+    {
+      "epoch": 11.04,
+      "learning_rate": 1.831578947368421e-05,
+      "loss": 5.007,
+      "step": 113000
+    },
+    {
+      "epoch": 11.14,
+      "learning_rate": 1.810526315789474e-05,
+      "loss": 5.0059,
+      "step": 114000
+    },
+    {
+      "epoch": 11.24,
+      "learning_rate": 1.7894736842105264e-05,
+      "loss": 5.0089,
+      "step": 115000
+    },
+    {
+      "epoch": 11.33,
+      "learning_rate": 1.768421052631579e-05,
+      "loss": 5.0081,
+      "step": 116000
+    },
+    {
+      "epoch": 11.43,
+      "learning_rate": 1.7473684210526317e-05,
+      "loss": 5.002,
+      "step": 117000
+    },
+    {
+      "epoch": 11.53,
+      "learning_rate": 1.7263157894736843e-05,
+      "loss": 5.0052,
+      "step": 118000
+    },
+    {
+      "epoch": 11.63,
+      "learning_rate": 1.705263157894737e-05,
+      "loss": 5.0079,
+      "step": 119000
+    },
+    {
+      "epoch": 11.72,
+      "learning_rate": 1.6842105263157896e-05,
+      "loss": 4.9985,
+      "step": 120000
+    },
+    {
+      "epoch": 11.82,
+      "learning_rate": 1.6631578947368423e-05,
+      "loss": 5.006,
+      "step": 121000
+    },
+    {
+      "epoch": 11.92,
+      "learning_rate": 1.642105263157895e-05,
+      "loss": 4.9956,
+      "step": 122000
+    },
+    {
+      "epoch": 12.02,
+      "learning_rate": 1.6210526315789473e-05,
+      "loss": 4.9973,
+      "step": 123000
+    },
+    {
+      "epoch": 12.12,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 4.9976,
+      "step": 124000
+    },
+    {
+      "epoch": 12.21,
+      "learning_rate": 1.578947368421053e-05,
+      "loss": 4.9882,
+      "step": 125000
+    },
+    {
+      "epoch": 12.31,
+      "learning_rate": 1.5578947368421052e-05,
+      "loss": 4.9908,
+      "step": 126000
+    },
+    {
+      "epoch": 12.41,
+      "learning_rate": 1.536842105263158e-05,
+      "loss": 5.0038,
+      "step": 127000
+    },
+    {
+      "epoch": 12.51,
+      "learning_rate": 1.5157894736842107e-05,
+      "loss": 4.9923,
+      "step": 128000
+    },
+    {
+      "epoch": 12.6,
+      "learning_rate": 1.4947368421052632e-05,
+      "loss": 4.9847,
+      "step": 129000
+    },
+    {
+      "epoch": 12.7,
+      "learning_rate": 1.4736842105263159e-05,
+      "loss": 4.9864,
+      "step": 130000
+    },
+    {
+      "epoch": 12.8,
+      "learning_rate": 1.4526315789473687e-05,
+      "loss": 4.9903,
+      "step": 131000
+    },
+    {
+      "epoch": 12.9,
+      "learning_rate": 1.4315789473684212e-05,
+      "loss": 4.9812,
+      "step": 132000
+    },
+    {
+      "epoch": 12.99,
+      "learning_rate": 1.4105263157894738e-05,
+      "loss": 4.9923,
+      "step": 133000
+    },
+    {
+      "epoch": 13.09,
+      "learning_rate": 1.3894736842105265e-05,
+      "loss": 4.9838,
+      "step": 134000
+    },
+    {
+      "epoch": 13.19,
+      "learning_rate": 1.3684210526315791e-05,
+      "loss": 4.977,
+      "step": 135000
+    },
+    {
+      "epoch": 13.29,
+      "learning_rate": 1.3473684210526316e-05,
+      "loss": 4.9743,
+      "step": 136000
+    },
+    {
+      "epoch": 13.39,
+      "learning_rate": 1.3263157894736843e-05,
+      "loss": 4.9797,
+      "step": 137000
+    },
+    {
+      "epoch": 13.48,
+      "learning_rate": 1.305263157894737e-05,
+      "loss": 4.9915,
+      "step": 138000
+    },
+    {
+      "epoch": 13.58,
+      "learning_rate": 1.2842105263157896e-05,
+      "loss": 4.9762,
+      "step": 139000
+    },
+    {
+      "epoch": 13.68,
+      "learning_rate": 1.263157894736842e-05,
+      "loss": 4.9771,
+      "step": 140000
+    },
+    {
+      "epoch": 13.78,
+      "learning_rate": 1.2421052631578949e-05,
+      "loss": 4.9763,
+      "step": 141000
+    },
+    {
+      "epoch": 13.87,
+      "learning_rate": 1.2210526315789475e-05,
+      "loss": 4.9706,
+      "step": 142000
+    },
+    {
+      "epoch": 13.97,
+      "learning_rate": 1.2e-05,
+      "loss": 4.9731,
+      "step": 143000
+    },
+    {
+      "epoch": 14.07,
+      "learning_rate": 1.1789473684210527e-05,
+      "loss": 4.9736,
+      "step": 144000
+    },
+    {
+      "epoch": 14.17,
+      "learning_rate": 1.1578947368421053e-05,
+      "loss": 4.9704,
+      "step": 145000
+    },
+    {
+      "epoch": 14.26,
+      "learning_rate": 1.136842105263158e-05,
+      "loss": 4.9762,
+      "step": 146000
+    },
+    {
+      "epoch": 14.36,
+      "learning_rate": 1.1157894736842105e-05,
+      "loss": 4.9709,
+      "step": 147000
+    },
+    {
+      "epoch": 14.46,
+      "learning_rate": 1.0947368421052633e-05,
+      "loss": 4.957,
+      "step": 148000
+    },
+    {
+      "epoch": 14.56,
+      "learning_rate": 1.073684210526316e-05,
+      "loss": 4.9666,
+      "step": 149000
+    },
+    {
+      "epoch": 14.66,
+      "learning_rate": 1.0526315789473684e-05,
+      "loss": 4.9623,
+      "step": 150000
+    },
+    {
+      "epoch": 14.75,
+      "learning_rate": 1.0315789473684213e-05,
+      "loss": 4.9613,
+      "step": 151000
+    },
+    {
+      "epoch": 14.85,
+      "learning_rate": 1.0105263157894738e-05,
+      "loss": 4.9642,
+      "step": 152000
+    },
+    {
+      "epoch": 14.95,
+      "learning_rate": 9.894736842105264e-06,
+      "loss": 4.9605,
+      "step": 153000
+    },
+    {
+      "epoch": 15.05,
+      "learning_rate": 9.68421052631579e-06,
+      "loss": 4.9688,
+      "step": 154000
+    },
+    {
+      "epoch": 15.14,
+      "learning_rate": 9.473684210526315e-06,
+      "loss": 4.9635,
+      "step": 155000
+    },
+    {
+      "epoch": 15.24,
+      "learning_rate": 9.263157894736842e-06,
+      "loss": 4.9587,
+      "step": 156000
+    },
+    {
+      "epoch": 15.34,
+      "learning_rate": 9.05263157894737e-06,
+      "loss": 4.9633,
+      "step": 157000
+    },
+    {
+      "epoch": 15.44,
+      "learning_rate": 8.842105263157895e-06,
+      "loss": 4.9613,
+      "step": 158000
+    },
+    {
+      "epoch": 15.53,
+      "learning_rate": 8.631578947368422e-06,
+      "loss": 4.9629,
+      "step": 159000
+    },
+    {
+      "epoch": 15.63,
+      "learning_rate": 8.421052631578948e-06,
+      "loss": 4.9547,
+      "step": 160000
+    },
+    {
+      "epoch": 15.73,
+      "learning_rate": 8.210526315789475e-06,
+      "loss": 4.9541,
+      "step": 161000
+    },
+    {
+      "epoch": 15.83,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 4.9532,
+      "step": 162000
+    },
+    {
+      "epoch": 15.93,
+      "learning_rate": 7.789473684210526e-06,
+      "loss": 4.9461,
+      "step": 163000
+    },
+    {
+      "epoch": 16.02,
+      "learning_rate": 7.578947368421054e-06,
+      "loss": 4.9518,
+      "step": 164000
+    },
+    {
+      "epoch": 16.12,
+      "learning_rate": 7.368421052631579e-06,
+      "loss": 4.9543,
+      "step": 165000
+    },
+    {
+      "epoch": 16.22,
+      "learning_rate": 7.157894736842106e-06,
+      "loss": 4.9526,
+      "step": 166000
+    },
+    {
+      "epoch": 16.32,
+      "learning_rate": 6.947368421052632e-06,
+      "loss": 4.9447,
+      "step": 167000
+    },
+    {
+      "epoch": 16.41,
+      "learning_rate": 6.736842105263158e-06,
+      "loss": 4.9545,
+      "step": 168000
+    },
+    {
+      "epoch": 16.51,
+      "learning_rate": 6.526315789473685e-06,
+      "loss": 4.9448,
+      "step": 169000
+    },
+    {
+      "epoch": 16.61,
+      "learning_rate": 6.31578947368421e-06,
+      "loss": 4.9521,
+      "step": 170000
+    },
+    {
+      "epoch": 16.71,
+      "learning_rate": 6.105263157894738e-06,
+      "loss": 4.9503,
+      "step": 171000
+    },
+    {
+      "epoch": 16.81,
+      "learning_rate": 5.8947368421052634e-06,
+      "loss": 4.9537,
+      "step": 172000
+    },
+    {
+      "epoch": 16.9,
+      "learning_rate": 5.68421052631579e-06,
+      "loss": 4.9446,
+      "step": 173000
+    },
+    {
+      "epoch": 17.0,
+      "learning_rate": 5.4736842105263165e-06,
+      "loss": 4.9397,
+      "step": 174000
+    },
+    {
+      "epoch": 17.1,
+      "learning_rate": 5.263157894736842e-06,
+      "loss": 4.9541,
+      "step": 175000
+    },
+    {
+      "epoch": 17.2,
+      "learning_rate": 5.052631578947369e-06,
+      "loss": 4.9449,
+      "step": 176000
+    },
+    {
+      "epoch": 17.29,
+      "learning_rate": 4.842105263157895e-06,
+      "loss": 4.9447,
+      "step": 177000
+    },
+    {
+      "epoch": 17.39,
+      "learning_rate": 4.631578947368421e-06,
+      "loss": 4.9344,
+      "step": 178000
+    },
+    {
+      "epoch": 17.49,
+      "learning_rate": 4.4210526315789476e-06,
+      "loss": 4.933,
+      "step": 179000
+    },
+    {
+      "epoch": 17.59,
+      "learning_rate": 4.210526315789474e-06,
+      "loss": 4.9397,
+      "step": 180000
+    },
+    {
+      "epoch": 17.68,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 4.9431,
+      "step": 181000
+    },
+    {
+      "epoch": 17.78,
+      "learning_rate": 3.789473684210527e-06,
+      "loss": 4.9378,
+      "step": 182000
+    },
+    {
+      "epoch": 17.88,
+      "learning_rate": 3.578947368421053e-06,
+      "loss": 4.9399,
+      "step": 183000
+    },
+    {
+      "epoch": 17.98,
+      "learning_rate": 3.368421052631579e-06,
+      "loss": 4.9481,
+      "step": 184000
+    },
+    {
+      "epoch": 18.08,
+      "learning_rate": 3.157894736842105e-06,
+      "loss": 4.9399,
+      "step": 185000
+    },
+    {
+      "epoch": 18.17,
+      "learning_rate": 2.9473684210526317e-06,
+      "loss": 4.9365,
+      "step": 186000
+    },
+    {
+      "epoch": 18.27,
+      "learning_rate": 2.7368421052631583e-06,
+      "loss": 4.9307,
+      "step": 187000
+    },
+    {
+      "epoch": 18.37,
+      "learning_rate": 2.5263157894736844e-06,
+      "loss": 4.9391,
+      "step": 188000
+    },
+    {
+      "epoch": 18.47,
+      "learning_rate": 2.3157894736842105e-06,
+      "loss": 4.9331,
+      "step": 189000
+    },
+    {
+      "epoch": 18.56,
+      "learning_rate": 2.105263157894737e-06,
+      "loss": 4.948,
+      "step": 190000
+    },
+    {
+      "epoch": 18.66,
+      "learning_rate": 1.8947368421052634e-06,
+      "loss": 4.9407,
+      "step": 191000
+    }
+  ],
+  "logging_steps": 1000,
+  "max_steps": 200000,
+  "num_train_epochs": 20,
+  "save_steps": 1000,
+  "total_flos": 2.250667778458596e+19,
+  "trial_name": null,
+  "trial_params": null
+}

len2_5120/checkpoint-191000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9ff5d6388a59708cb6305a7fab984d438e385b1835680f9629b90c23c5cf704
+size 5137

len2_5120/checkpoint-192000/config.json

@@ -0,0 +1,25 @@
+{
+  "architectures": [
+    "BertForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_length": 512,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.35.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 5120
+}

len2_5120/checkpoint-192000/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "max_length": 512,
+  "pad_token_id": 0,
+  "transformers_version": "4.35.2"
+}

len2_5120/checkpoint-192000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4cbffd267340e7a09c8e1143725cb945c07c0c1658055e0fba16d5ae1b14c7ac
+size 359944256

len2_5120/checkpoint-192000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06d737a339b674a47bbb393421311b07894bc8deb11a35745119f729e10925b8
+size 720010699

len2_5120/checkpoint-192000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94f051d28f4c9a4e5765ca63afa821d32882244885368ce9c7547bfc4cd15fb2
+size 14645

len2_5120/checkpoint-192000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8987c093fe567c44cd8f1b73e4b15e5d4b9c5495e847cc1b11a0915470ccf64f
+size 1465

len2_5120/checkpoint-192000/trainer_state.json

@@ -0,0 +1,1171 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 18.75915974596971,
+  "eval_steps": 500,
+  "global_step": 192000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.78,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 7.5832,
+      "step": 1000
+    },
+    {
+      "epoch": 1.56,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 6.9724,
+      "step": 2000
+    },
+    {
+      "epoch": 2.34,
+      "learning_rate": 1.2e-05,
+      "loss": 6.7695,
+      "step": 3000
+    },
+    {
+      "epoch": 3.12,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 6.7002,
+      "step": 4000
+    },
+    {
+      "epoch": 3.91,
+      "learning_rate": 2e-05,
+      "loss": 6.666,
+      "step": 5000
+    },
+    {
+      "epoch": 4.69,
+      "learning_rate": 2.4e-05,
+      "loss": 6.6503,
+      "step": 6000
+    },
+    {
+      "epoch": 5.47,
+      "learning_rate": 2.8e-05,
+      "loss": 6.6407,
+      "step": 7000
+    },
+    {
+      "epoch": 6.25,
+      "learning_rate": 3.2000000000000005e-05,
+      "loss": 6.6277,
+      "step": 8000
+    },
+    {
+      "epoch": 7.03,
+      "learning_rate": 3.6e-05,
+      "loss": 6.6209,
+      "step": 9000
+    },
+    {
+      "epoch": 7.81,
+      "learning_rate": 4e-05,
+      "loss": 6.6135,
+      "step": 10000
+    },
+    {
+      "epoch": 8.59,
+      "learning_rate": 3.955555555555556e-05,
+      "loss": 6.6084,
+      "step": 11000
+    },
+    {
+      "epoch": 9.38,
+      "learning_rate": 3.9111111111111115e-05,
+      "loss": 6.6041,
+      "step": 12000
+    },
+    {
+      "epoch": 10.16,
+      "learning_rate": 3.866666666666667e-05,
+      "loss": 6.6013,
+      "step": 13000
+    },
+    {
+      "epoch": 10.94,
+      "learning_rate": 3.8222222222222226e-05,
+      "loss": 6.5994,
+      "step": 14000
+    },
+    {
+      "epoch": 11.72,
+      "learning_rate": 3.777777777777778e-05,
+      "loss": 6.594,
+      "step": 15000
+    },
+    {
+      "epoch": 12.5,
+      "learning_rate": 3.733333333333334e-05,
+      "loss": 6.5925,
+      "step": 16000
+    },
+    {
+      "epoch": 13.28,
+      "learning_rate": 3.6888888888888896e-05,
+      "loss": 6.5893,
+      "step": 17000
+    },
+    {
+      "epoch": 14.06,
+      "learning_rate": 3.644444444444445e-05,
+      "loss": 6.5875,
+      "step": 18000
+    },
+    {
+      "epoch": 14.84,
+      "learning_rate": 3.6e-05,
+      "loss": 6.5232,
+      "step": 19000
+    },
+    {
+      "epoch": 15.62,
+      "learning_rate": 3.555555555555555e-05,
+      "loss": 6.4358,
+      "step": 20000
+    },
+    {
+      "epoch": 16.41,
+      "learning_rate": 3.511111111111111e-05,
+      "loss": 6.3712,
+      "step": 21000
+    },
+    {
+      "epoch": 17.19,
+      "learning_rate": 3.466666666666667e-05,
+      "loss": 6.311,
+      "step": 22000
+    },
+    {
+      "epoch": 17.97,
+      "learning_rate": 3.4222222222222224e-05,
+      "loss": 6.2493,
+      "step": 23000
+    },
+    {
+      "epoch": 18.75,
+      "learning_rate": 3.377777777777778e-05,
+      "loss": 6.1597,
+      "step": 24000
+    },
+    {
+      "epoch": 19.53,
+      "learning_rate": 3.3333333333333335e-05,
+      "loss": 6.0276,
+      "step": 25000
+    },
+    {
+      "epoch": 20.31,
+      "learning_rate": 3.288888888888889e-05,
+      "loss": 5.8317,
+      "step": 26000
+    },
+    {
+      "epoch": 21.09,
+      "learning_rate": 3.2444444444444446e-05,
+      "loss": 5.6782,
+      "step": 27000
+    },
+    {
+      "epoch": 21.88,
+      "learning_rate": 3.2000000000000005e-05,
+      "loss": 5.5975,
+      "step": 28000
+    },
+    {
+      "epoch": 22.66,
+      "learning_rate": 3.155555555555556e-05,
+      "loss": 5.5369,
+      "step": 29000
+    },
+    {
+      "epoch": 23.44,
+      "learning_rate": 3.111111111111112e-05,
+      "loss": 5.4921,
+      "step": 30000
+    },
+    {
+      "epoch": 24.22,
+      "learning_rate": 3.066666666666667e-05,
+      "loss": 5.4554,
+      "step": 31000
+    },
+    {
+      "epoch": 25.0,
+      "learning_rate": 3.0222222222222225e-05,
+      "loss": 5.4237,
+      "step": 32000
+    },
+    {
+      "epoch": 25.78,
+      "learning_rate": 2.977777777777778e-05,
+      "loss": 5.3938,
+      "step": 33000
+    },
+    {
+      "epoch": 26.56,
+      "learning_rate": 2.9333333333333333e-05,
+      "loss": 5.3701,
+      "step": 34000
+    },
+    {
+      "epoch": 27.34,
+      "learning_rate": 2.888888888888889e-05,
+      "loss": 5.3452,
+      "step": 35000
+    },
+    {
+      "epoch": 28.12,
+      "learning_rate": 2.8444444444444447e-05,
+      "loss": 5.327,
+      "step": 36000
+    },
+    {
+      "epoch": 28.91,
+      "learning_rate": 2.8e-05,
+      "loss": 5.3063,
+      "step": 37000
+    },
+    {
+      "epoch": 29.69,
+      "learning_rate": 2.755555555555556e-05,
+      "loss": 5.2905,
+      "step": 38000
+    },
+    {
+      "epoch": 30.47,
+      "learning_rate": 2.7111111111111114e-05,
+      "loss": 5.2747,
+      "step": 39000
+    },
+    {
+      "epoch": 31.25,
+      "learning_rate": 2.6666666666666667e-05,
+      "loss": 5.2596,
+      "step": 40000
+    },
+    {
+      "epoch": 32.03,
+      "learning_rate": 2.6222222222222226e-05,
+      "loss": 5.2483,
+      "step": 41000
+    },
+    {
+      "epoch": 32.81,
+      "learning_rate": 2.577777777777778e-05,
+      "loss": 5.2336,
+      "step": 42000
+    },
+    {
+      "epoch": 33.59,
+      "learning_rate": 2.5333333333333334e-05,
+      "loss": 5.2214,
+      "step": 43000
+    },
+    {
+      "epoch": 34.38,
+      "learning_rate": 2.4888888888888893e-05,
+      "loss": 5.2103,
+      "step": 44000
+    },
+    {
+      "epoch": 35.16,
+      "learning_rate": 2.444444444444445e-05,
+      "loss": 5.2015,
+      "step": 45000
+    },
+    {
+      "epoch": 35.94,
+      "learning_rate": 2.4e-05,
+      "loss": 5.1907,
+      "step": 46000
+    },
+    {
+      "epoch": 36.72,
+      "learning_rate": 2.355555555555556e-05,
+      "loss": 5.1801,
+      "step": 47000
+    },
+    {
+      "epoch": 37.5,
+      "learning_rate": 2.3111111111111112e-05,
+      "loss": 5.1713,
+      "step": 48000
+    },
+    {
+      "epoch": 38.28,
+      "learning_rate": 2.2666666666666668e-05,
+      "loss": 5.1645,
+      "step": 49000
+    },
+    {
+      "epoch": 39.06,
+      "learning_rate": 2.2222222222222227e-05,
+      "loss": 5.1548,
+      "step": 50000
+    },
+    {
+      "epoch": 39.84,
+      "learning_rate": 2.177777777777778e-05,
+      "loss": 5.1462,
+      "step": 51000
+    },
+    {
+      "epoch": 40.62,
+      "learning_rate": 2.1333333333333335e-05,
+      "loss": 5.1417,
+      "step": 52000
+    },
+    {
+      "epoch": 41.41,
+      "learning_rate": 2.088888888888889e-05,
+      "loss": 5.1305,
+      "step": 53000
+    },
+    {
+      "epoch": 42.19,
+      "learning_rate": 2.0444444444444446e-05,
+      "loss": 5.1287,
+      "step": 54000
+    },
+    {
+      "epoch": 42.97,
+      "learning_rate": 2e-05,
+      "loss": 5.1202,
+      "step": 55000
+    },
+    {
+      "epoch": 43.75,
+      "learning_rate": 1.9555555555555557e-05,
+      "loss": 5.1139,
+      "step": 56000
+    },
+    {
+      "epoch": 44.53,
+      "learning_rate": 1.9111111111111113e-05,
+      "loss": 5.1081,
+      "step": 57000
+    },
+    {
+      "epoch": 45.31,
+      "learning_rate": 1.866666666666667e-05,
+      "loss": 5.1032,
+      "step": 58000
+    },
+    {
+      "epoch": 46.09,
+      "learning_rate": 1.8222222222222224e-05,
+      "loss": 5.0968,
+      "step": 59000
+    },
+    {
+      "epoch": 46.88,
+      "learning_rate": 1.7777777777777777e-05,
+      "loss": 5.0914,
+      "step": 60000
+    },
+    {
+      "epoch": 47.66,
+      "learning_rate": 1.7333333333333336e-05,
+      "loss": 5.085,
+      "step": 61000
+    },
+    {
+      "epoch": 48.44,
+      "learning_rate": 1.688888888888889e-05,
+      "loss": 5.0806,
+      "step": 62000
+    },
+    {
+      "epoch": 49.22,
+      "learning_rate": 1.6444444444444444e-05,
+      "loss": 5.0769,
+      "step": 63000
+    },
+    {
+      "epoch": 50.0,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 5.0728,
+      "step": 64000
+    },
+    {
+      "epoch": 50.78,
+      "learning_rate": 1.555555555555556e-05,
+      "loss": 5.0692,
+      "step": 65000
+    },
+    {
+      "epoch": 51.56,
+      "learning_rate": 1.5111111111111112e-05,
+      "loss": 5.065,
+      "step": 66000
+    },
+    {
+      "epoch": 52.34,
+      "learning_rate": 1.4666666666666666e-05,
+      "loss": 5.0594,
+      "step": 67000
+    },
+    {
+      "epoch": 53.12,
+      "learning_rate": 1.4222222222222224e-05,
+      "loss": 5.0549,
+      "step": 68000
+    },
+    {
+      "epoch": 53.91,
+      "learning_rate": 1.377777777777778e-05,
+      "loss": 5.0538,
+      "step": 69000
+    },
+    {
+      "epoch": 54.69,
+      "learning_rate": 1.3333333333333333e-05,
+      "loss": 5.0482,
+      "step": 70000
+    },
+    {
+      "epoch": 55.47,
+      "learning_rate": 1.288888888888889e-05,
+      "loss": 5.0475,
+      "step": 71000
+    },
+    {
+      "epoch": 56.25,
+      "learning_rate": 1.2444444444444446e-05,
+      "loss": 5.0399,
+      "step": 72000
+    },
+    {
+      "epoch": 57.03,
+      "learning_rate": 1.2e-05,
+      "loss": 5.0406,
+      "step": 73000
+    },
+    {
+      "epoch": 57.81,
+      "learning_rate": 1.1555555555555556e-05,
+      "loss": 5.037,
+      "step": 74000
+    },
+    {
+      "epoch": 58.59,
+      "learning_rate": 1.1111111111111113e-05,
+      "loss": 5.0326,
+      "step": 75000
+    },
+    {
+      "epoch": 59.38,
+      "learning_rate": 1.0666666666666667e-05,
+      "loss": 5.031,
+      "step": 76000
+    },
+    {
+      "epoch": 60.16,
+      "learning_rate": 1.0222222222222223e-05,
+      "loss": 5.0291,
+      "step": 77000
+    },
+    {
+      "epoch": 60.94,
+      "learning_rate": 9.777777777777779e-06,
+      "loss": 5.0266,
+      "step": 78000
+    },
+    {
+      "epoch": 61.72,
+      "learning_rate": 9.333333333333334e-06,
+      "loss": 5.0235,
+      "step": 79000
+    },
+    {
+      "epoch": 62.5,
+      "learning_rate": 8.888888888888888e-06,
+      "loss": 5.0211,
+      "step": 80000
+    },
+    {
+      "epoch": 63.28,
+      "learning_rate": 8.444444444444446e-06,
+      "loss": 5.0214,
+      "step": 81000
+    },
+    {
+      "epoch": 64.06,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 5.0176,
+      "step": 82000
+    },
+    {
+      "epoch": 64.84,
+      "learning_rate": 7.555555555555556e-06,
+      "loss": 5.0157,
+      "step": 83000
+    },
+    {
+      "epoch": 65.62,
+      "learning_rate": 7.111111111111112e-06,
+      "loss": 5.014,
+      "step": 84000
+    },
+    {
+      "epoch": 66.41,
+      "learning_rate": 6.666666666666667e-06,
+      "loss": 5.0115,
+      "step": 85000
+    },
+    {
+      "epoch": 67.19,
+      "learning_rate": 6.222222222222223e-06,
+      "loss": 5.0087,
+      "step": 86000
+    },
+    {
+      "epoch": 67.97,
+      "learning_rate": 5.777777777777778e-06,
+      "loss": 5.0105,
+      "step": 87000
+    },
+    {
+      "epoch": 68.75,
+      "learning_rate": 5.333333333333334e-06,
+      "loss": 5.0062,
+      "step": 88000
+    },
+    {
+      "epoch": 69.53,
+      "learning_rate": 4.888888888888889e-06,
+      "loss": 5.0028,
+      "step": 89000
+    },
+    {
+      "epoch": 70.31,
+      "learning_rate": 4.444444444444444e-06,
+      "loss": 5.0066,
+      "step": 90000
+    },
+    {
+      "epoch": 71.09,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 5.0032,
+      "step": 91000
+    },
+    {
+      "epoch": 71.88,
+      "learning_rate": 3.555555555555556e-06,
+      "loss": 5.0022,
+      "step": 92000
+    },
+    {
+      "epoch": 72.66,
+      "learning_rate": 3.1111111111111116e-06,
+      "loss": 5.0024,
+      "step": 93000
+    },
+    {
+      "epoch": 73.44,
+      "learning_rate": 2.666666666666667e-06,
+      "loss": 5.0007,
+      "step": 94000
+    },
+    {
+      "epoch": 74.22,
+      "learning_rate": 2.222222222222222e-06,
+      "loss": 5.0008,
+      "step": 95000
+    },
+    {
+      "epoch": 75.0,
+      "learning_rate": 1.777777777777778e-06,
+      "loss": 4.9978,
+      "step": 96000
+    },
+    {
+      "epoch": 75.78,
+      "learning_rate": 1.3333333333333334e-06,
+      "loss": 4.9966,
+      "step": 97000
+    },
+    {
+      "epoch": 76.56,
+      "learning_rate": 8.88888888888889e-07,
+      "loss": 4.9994,
+      "step": 98000
+    },
+    {
+      "epoch": 77.34,
+      "learning_rate": 4.444444444444445e-07,
+      "loss": 4.9966,
+      "step": 99000
+    },
+    {
+      "epoch": 78.12,
+      "learning_rate": 0.0,
+      "loss": 4.9976,
+      "step": 100000
+    },
+    {
+      "epoch": 9.87,
+      "learning_rate": 2.0842105263157895e-05,
+      "loss": 5.0155,
+      "step": 101000
+    },
+    {
+      "epoch": 9.97,
+      "learning_rate": 2.0631578947368425e-05,
+      "loss": 5.024,
+      "step": 102000
+    },
+    {
+      "epoch": 10.06,
+      "learning_rate": 2.0421052631578952e-05,
+      "loss": 5.0241,
+      "step": 103000
+    },
+    {
+      "epoch": 10.16,
+      "learning_rate": 2.0210526315789475e-05,
+      "loss": 5.0218,
+      "step": 104000
+    },
+    {
+      "epoch": 10.26,
+      "learning_rate": 2e-05,
+      "loss": 5.0231,
+      "step": 105000
+    },
+    {
+      "epoch": 10.36,
+      "learning_rate": 1.9789473684210528e-05,
+      "loss": 5.0231,
+      "step": 106000
+    },
+    {
+      "epoch": 10.45,
+      "learning_rate": 1.9578947368421055e-05,
+      "loss": 5.0198,
+      "step": 107000
+    },
+    {
+      "epoch": 10.55,
+      "learning_rate": 1.936842105263158e-05,
+      "loss": 5.0158,
+      "step": 108000
+    },
+    {
+      "epoch": 10.65,
+      "learning_rate": 1.9157894736842108e-05,
+      "loss": 5.0122,
+      "step": 109000
+    },
+    {
+      "epoch": 10.75,
+      "learning_rate": 1.894736842105263e-05,
+      "loss": 5.0165,
+      "step": 110000
+    },
+    {
+      "epoch": 10.85,
+      "learning_rate": 1.873684210526316e-05,
+      "loss": 5.0183,
+      "step": 111000
+    },
+    {
+      "epoch": 10.94,
+      "learning_rate": 1.8526315789473684e-05,
+      "loss": 5.01,
+      "step": 112000
+    },
+    {
+      "epoch": 11.04,
+      "learning_rate": 1.831578947368421e-05,
+      "loss": 5.007,
+      "step": 113000
+    },
+    {
+      "epoch": 11.14,
+      "learning_rate": 1.810526315789474e-05,
+      "loss": 5.0059,
+      "step": 114000
+    },
+    {
+      "epoch": 11.24,
+      "learning_rate": 1.7894736842105264e-05,
+      "loss": 5.0089,
+      "step": 115000
+    },
+    {
+      "epoch": 11.33,
+      "learning_rate": 1.768421052631579e-05,
+      "loss": 5.0081,
+      "step": 116000
+    },
+    {
+      "epoch": 11.43,
+      "learning_rate": 1.7473684210526317e-05,
+      "loss": 5.002,
+      "step": 117000
+    },
+    {
+      "epoch": 11.53,
+      "learning_rate": 1.7263157894736843e-05,
+      "loss": 5.0052,
+      "step": 118000
+    },
+    {
+      "epoch": 11.63,
+      "learning_rate": 1.705263157894737e-05,
+      "loss": 5.0079,
+      "step": 119000
+    },
+    {
+      "epoch": 11.72,
+      "learning_rate": 1.6842105263157896e-05,
+      "loss": 4.9985,
+      "step": 120000
+    },
+    {
+      "epoch": 11.82,
+      "learning_rate": 1.6631578947368423e-05,
+      "loss": 5.006,
+      "step": 121000
+    },
+    {
+      "epoch": 11.92,
+      "learning_rate": 1.642105263157895e-05,
+      "loss": 4.9956,
+      "step": 122000
+    },
+    {
+      "epoch": 12.02,
+      "learning_rate": 1.6210526315789473e-05,
+      "loss": 4.9973,
+      "step": 123000
+    },
+    {
+      "epoch": 12.12,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 4.9976,
+      "step": 124000
+    },
+    {
+      "epoch": 12.21,
+      "learning_rate": 1.578947368421053e-05,
+      "loss": 4.9882,
+      "step": 125000
+    },
+    {
+      "epoch": 12.31,
+      "learning_rate": 1.5578947368421052e-05,
+      "loss": 4.9908,
+      "step": 126000
+    },
+    {
+      "epoch": 12.41,
+      "learning_rate": 1.536842105263158e-05,
+      "loss": 5.0038,
+      "step": 127000
+    },
+    {
+      "epoch": 12.51,
+      "learning_rate": 1.5157894736842107e-05,
+      "loss": 4.9923,
+      "step": 128000
+    },
+    {
+      "epoch": 12.6,
+      "learning_rate": 1.4947368421052632e-05,
+      "loss": 4.9847,
+      "step": 129000
+    },
+    {
+      "epoch": 12.7,
+      "learning_rate": 1.4736842105263159e-05,
+      "loss": 4.9864,
+      "step": 130000
+    },
+    {
+      "epoch": 12.8,
+      "learning_rate": 1.4526315789473687e-05,
+      "loss": 4.9903,
+      "step": 131000
+    },
+    {
+      "epoch": 12.9,
+      "learning_rate": 1.4315789473684212e-05,
+      "loss": 4.9812,
+      "step": 132000
+    },
+    {
+      "epoch": 12.99,
+      "learning_rate": 1.4105263157894738e-05,
+      "loss": 4.9923,
+      "step": 133000
+    },
+    {
+      "epoch": 13.09,
+      "learning_rate": 1.3894736842105265e-05,
+      "loss": 4.9838,
+      "step": 134000
+    },
+    {
+      "epoch": 13.19,
+      "learning_rate": 1.3684210526315791e-05,
+      "loss": 4.977,
+      "step": 135000
+    },
+    {
+      "epoch": 13.29,
+      "learning_rate": 1.3473684210526316e-05,
+      "loss": 4.9743,
+      "step": 136000
+    },
+    {
+      "epoch": 13.39,
+      "learning_rate": 1.3263157894736843e-05,
+      "loss": 4.9797,
+      "step": 137000
+    },
+    {
+      "epoch": 13.48,
+      "learning_rate": 1.305263157894737e-05,
+      "loss": 4.9915,
+      "step": 138000
+    },
+    {
+      "epoch": 13.58,
+      "learning_rate": 1.2842105263157896e-05,
+      "loss": 4.9762,
+      "step": 139000
+    },
+    {
+      "epoch": 13.68,
+      "learning_rate": 1.263157894736842e-05,
+      "loss": 4.9771,
+      "step": 140000
+    },
+    {
+      "epoch": 13.78,
+      "learning_rate": 1.2421052631578949e-05,
+      "loss": 4.9763,
+      "step": 141000
+    },
+    {
+      "epoch": 13.87,
+      "learning_rate": 1.2210526315789475e-05,
+      "loss": 4.9706,
+      "step": 142000
+    },
+    {
+      "epoch": 13.97,
+      "learning_rate": 1.2e-05,
+      "loss": 4.9731,
+      "step": 143000
+    },
+    {
+      "epoch": 14.07,
+      "learning_rate": 1.1789473684210527e-05,
+      "loss": 4.9736,
+      "step": 144000
+    },
+    {
+      "epoch": 14.17,
+      "learning_rate": 1.1578947368421053e-05,
+      "loss": 4.9704,
+      "step": 145000
+    },
+    {
+      "epoch": 14.26,
+      "learning_rate": 1.136842105263158e-05,
+      "loss": 4.9762,
+      "step": 146000
+    },
+    {
+      "epoch": 14.36,
+      "learning_rate": 1.1157894736842105e-05,
+      "loss": 4.9709,
+      "step": 147000
+    },
+    {
+      "epoch": 14.46,
+      "learning_rate": 1.0947368421052633e-05,
+      "loss": 4.957,
+      "step": 148000
+    },
+    {
+      "epoch": 14.56,
+      "learning_rate": 1.073684210526316e-05,
+      "loss": 4.9666,
+      "step": 149000
+    },
+    {
+      "epoch": 14.66,
+      "learning_rate": 1.0526315789473684e-05,
+      "loss": 4.9623,
+      "step": 150000
+    },
+    {
+      "epoch": 14.75,
+      "learning_rate": 1.0315789473684213e-05,
+      "loss": 4.9613,
+      "step": 151000
+    },
+    {
+      "epoch": 14.85,
+      "learning_rate": 1.0105263157894738e-05,
+      "loss": 4.9642,
+      "step": 152000
+    },
+    {
+      "epoch": 14.95,
+      "learning_rate": 9.894736842105264e-06,
+      "loss": 4.9605,
+      "step": 153000
+    },
+    {
+      "epoch": 15.05,
+      "learning_rate": 9.68421052631579e-06,
+      "loss": 4.9688,
+      "step": 154000
+    },
+    {
+      "epoch": 15.14,
+      "learning_rate": 9.473684210526315e-06,
+      "loss": 4.9635,
+      "step": 155000
+    },
+    {
+      "epoch": 15.24,
+      "learning_rate": 9.263157894736842e-06,
+      "loss": 4.9587,
+      "step": 156000
+    },
+    {
+      "epoch": 15.34,
+      "learning_rate": 9.05263157894737e-06,
+      "loss": 4.9633,
+      "step": 157000
+    },
+    {
+      "epoch": 15.44,
+      "learning_rate": 8.842105263157895e-06,
+      "loss": 4.9613,
+      "step": 158000
+    },
+    {
+      "epoch": 15.53,
+      "learning_rate": 8.631578947368422e-06,
+      "loss": 4.9629,
+      "step": 159000
+    },
+    {
+      "epoch": 15.63,
+      "learning_rate": 8.421052631578948e-06,
+      "loss": 4.9547,
+      "step": 160000
+    },
+    {
+      "epoch": 15.73,
+      "learning_rate": 8.210526315789475e-06,
+      "loss": 4.9541,
+      "step": 161000
+    },
+    {
+      "epoch": 15.83,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 4.9532,
+      "step": 162000
+    },
+    {
+      "epoch": 15.93,
+      "learning_rate": 7.789473684210526e-06,
+      "loss": 4.9461,
+      "step": 163000
+    },
+    {
+      "epoch": 16.02,
+      "learning_rate": 7.578947368421054e-06,
+      "loss": 4.9518,
+      "step": 164000
+    },
+    {
+      "epoch": 16.12,
+      "learning_rate": 7.368421052631579e-06,
+      "loss": 4.9543,
+      "step": 165000
+    },
+    {
+      "epoch": 16.22,
+      "learning_rate": 7.157894736842106e-06,
+      "loss": 4.9526,
+      "step": 166000
+    },
+    {
+      "epoch": 16.32,
+      "learning_rate": 6.947368421052632e-06,
+      "loss": 4.9447,
+      "step": 167000
+    },
+    {
+      "epoch": 16.41,
+      "learning_rate": 6.736842105263158e-06,
+      "loss": 4.9545,
+      "step": 168000
+    },
+    {
+      "epoch": 16.51,
+      "learning_rate": 6.526315789473685e-06,
+      "loss": 4.9448,
+      "step": 169000
+    },
+    {
+      "epoch": 16.61,
+      "learning_rate": 6.31578947368421e-06,
+      "loss": 4.9521,
+      "step": 170000
+    },
+    {
+      "epoch": 16.71,
+      "learning_rate": 6.105263157894738e-06,
+      "loss": 4.9503,
+      "step": 171000
+    },
+    {
+      "epoch": 16.81,
+      "learning_rate": 5.8947368421052634e-06,
+      "loss": 4.9537,
+      "step": 172000
+    },
+    {
+      "epoch": 16.9,
+      "learning_rate": 5.68421052631579e-06,
+      "loss": 4.9446,
+      "step": 173000
+    },
+    {
+      "epoch": 17.0,
+      "learning_rate": 5.4736842105263165e-06,
+      "loss": 4.9397,
+      "step": 174000
+    },
+    {
+      "epoch": 17.1,
+      "learning_rate": 5.263157894736842e-06,
+      "loss": 4.9541,
+      "step": 175000
+    },
+    {
+      "epoch": 17.2,
+      "learning_rate": 5.052631578947369e-06,
+      "loss": 4.9449,
+      "step": 176000
+    },
+    {
+      "epoch": 17.29,
+      "learning_rate": 4.842105263157895e-06,
+      "loss": 4.9447,
+      "step": 177000
+    },
+    {
+      "epoch": 17.39,
+      "learning_rate": 4.631578947368421e-06,
+      "loss": 4.9344,
+      "step": 178000
+    },
+    {
+      "epoch": 17.49,
+      "learning_rate": 4.4210526315789476e-06,
+      "loss": 4.933,
+      "step": 179000
+    },
+    {
+      "epoch": 17.59,
+      "learning_rate": 4.210526315789474e-06,
+      "loss": 4.9397,
+      "step": 180000
+    },
+    {
+      "epoch": 17.68,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 4.9431,
+      "step": 181000
+    },
+    {
+      "epoch": 17.78,
+      "learning_rate": 3.789473684210527e-06,
+      "loss": 4.9378,
+      "step": 182000
+    },
+    {
+      "epoch": 17.88,
+      "learning_rate": 3.578947368421053e-06,
+      "loss": 4.9399,
+      "step": 183000
+    },
+    {
+      "epoch": 17.98,
+      "learning_rate": 3.368421052631579e-06,
+      "loss": 4.9481,
+      "step": 184000
+    },
+    {
+      "epoch": 18.08,
+      "learning_rate": 3.157894736842105e-06,
+      "loss": 4.9399,
+      "step": 185000
+    },
+    {
+      "epoch": 18.17,
+      "learning_rate": 2.9473684210526317e-06,
+      "loss": 4.9365,
+      "step": 186000
+    },
+    {
+      "epoch": 18.27,
+      "learning_rate": 2.7368421052631583e-06,
+      "loss": 4.9307,
+      "step": 187000
+    },
+    {
+      "epoch": 18.37,
+      "learning_rate": 2.5263157894736844e-06,
+      "loss": 4.9391,
+      "step": 188000
+    },
+    {
+      "epoch": 18.47,
+      "learning_rate": 2.3157894736842105e-06,
+      "loss": 4.9331,
+      "step": 189000
+    },
+    {
+      "epoch": 18.56,
+      "learning_rate": 2.105263157894737e-06,
+      "loss": 4.948,
+      "step": 190000
+    },
+    {
+      "epoch": 18.66,
+      "learning_rate": 1.8947368421052634e-06,
+      "loss": 4.9407,
+      "step": 191000
+    },
+    {
+      "epoch": 18.76,
+      "learning_rate": 1.6842105263157895e-06,
+      "loss": 4.944,
+      "step": 192000
+    }
+  ],
+  "logging_steps": 1000,
+  "max_steps": 200000,
+  "num_train_epochs": 20,
+  "save_steps": 1000,
+  "total_flos": 2.2531937955260137e+19,
+  "trial_name": null,
+  "trial_params": null
+}

len2_5120/checkpoint-192000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9ff5d6388a59708cb6305a7fab984d438e385b1835680f9629b90c23c5cf704
+size 5137

len2_5120/checkpoint-193000/config.json

@@ -0,0 +1,25 @@
+{
+  "architectures": [
+    "BertForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_length": 512,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.35.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 5120
+}

len2_5120/checkpoint-193000/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "max_length": 512,
+  "pad_token_id": 0,
+  "transformers_version": "4.35.2"
+}

len2_5120/checkpoint-193000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2967019014b91b75c93d4a3368eade80f165f36a45202547cb4fffa4d2bd3dcc
+size 359944256

len2_5120/checkpoint-193000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe6d3dd22b82273c51f5e1ef00d4b8b80373bff065365e28532536ac9459fe50
+size 720010699

len2_5120/checkpoint-193000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a14699627c750f517769353fa39bf167eec026587c76244f12abfd0f16cabde3
+size 14645

len2_5120/checkpoint-193000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:193ff94d9f24279c5f8926a3af20ad4747ac16bb078d5e13b935e0b3be972c5b
+size 1465

len2_5120/checkpoint-193000/trainer_state.json

@@ -0,0 +1,1177 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 18.85686370297997,
+  "eval_steps": 500,
+  "global_step": 193000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.78,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 7.5832,
+      "step": 1000
+    },
+    {
+      "epoch": 1.56,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 6.9724,
+      "step": 2000
+    },
+    {
+      "epoch": 2.34,
+      "learning_rate": 1.2e-05,
+      "loss": 6.7695,
+      "step": 3000
+    },
+    {
+      "epoch": 3.12,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 6.7002,
+      "step": 4000
+    },
+    {
+      "epoch": 3.91,
+      "learning_rate": 2e-05,
+      "loss": 6.666,
+      "step": 5000
+    },
+    {
+      "epoch": 4.69,
+      "learning_rate": 2.4e-05,
+      "loss": 6.6503,
+      "step": 6000
+    },
+    {
+      "epoch": 5.47,
+      "learning_rate": 2.8e-05,
+      "loss": 6.6407,
+      "step": 7000
+    },
+    {
+      "epoch": 6.25,
+      "learning_rate": 3.2000000000000005e-05,
+      "loss": 6.6277,
+      "step": 8000
+    },
+    {
+      "epoch": 7.03,
+      "learning_rate": 3.6e-05,
+      "loss": 6.6209,
+      "step": 9000
+    },
+    {
+      "epoch": 7.81,
+      "learning_rate": 4e-05,
+      "loss": 6.6135,
+      "step": 10000
+    },
+    {
+      "epoch": 8.59,
+      "learning_rate": 3.955555555555556e-05,
+      "loss": 6.6084,
+      "step": 11000
+    },
+    {
+      "epoch": 9.38,
+      "learning_rate": 3.9111111111111115e-05,
+      "loss": 6.6041,
+      "step": 12000
+    },
+    {
+      "epoch": 10.16,
+      "learning_rate": 3.866666666666667e-05,
+      "loss": 6.6013,
+      "step": 13000
+    },
+    {
+      "epoch": 10.94,
+      "learning_rate": 3.8222222222222226e-05,
+      "loss": 6.5994,
+      "step": 14000
+    },
+    {
+      "epoch": 11.72,
+      "learning_rate": 3.777777777777778e-05,
+      "loss": 6.594,
+      "step": 15000
+    },
+    {
+      "epoch": 12.5,
+      "learning_rate": 3.733333333333334e-05,
+      "loss": 6.5925,
+      "step": 16000
+    },
+    {
+      "epoch": 13.28,
+      "learning_rate": 3.6888888888888896e-05,
+      "loss": 6.5893,
+      "step": 17000
+    },
+    {
+      "epoch": 14.06,
+      "learning_rate": 3.644444444444445e-05,
+      "loss": 6.5875,
+      "step": 18000
+    },
+    {
+      "epoch": 14.84,
+      "learning_rate": 3.6e-05,
+      "loss": 6.5232,
+      "step": 19000
+    },
+    {
+      "epoch": 15.62,
+      "learning_rate": 3.555555555555555e-05,
+      "loss": 6.4358,
+      "step": 20000
+    },
+    {
+      "epoch": 16.41,
+      "learning_rate": 3.511111111111111e-05,
+      "loss": 6.3712,
+      "step": 21000
+    },
+    {
+      "epoch": 17.19,
+      "learning_rate": 3.466666666666667e-05,
+      "loss": 6.311,
+      "step": 22000
+    },
+    {
+      "epoch": 17.97,
+      "learning_rate": 3.4222222222222224e-05,
+      "loss": 6.2493,
+      "step": 23000
+    },
+    {
+      "epoch": 18.75,
+      "learning_rate": 3.377777777777778e-05,
+      "loss": 6.1597,
+      "step": 24000
+    },
+    {
+      "epoch": 19.53,
+      "learning_rate": 3.3333333333333335e-05,
+      "loss": 6.0276,
+      "step": 25000
+    },
+    {
+      "epoch": 20.31,
+      "learning_rate": 3.288888888888889e-05,
+      "loss": 5.8317,
+      "step": 26000
+    },
+    {
+      "epoch": 21.09,
+      "learning_rate": 3.2444444444444446e-05,
+      "loss": 5.6782,
+      "step": 27000
+    },
+    {
+      "epoch": 21.88,
+      "learning_rate": 3.2000000000000005e-05,
+      "loss": 5.5975,
+      "step": 28000
+    },
+    {
+      "epoch": 22.66,
+      "learning_rate": 3.155555555555556e-05,
+      "loss": 5.5369,
+      "step": 29000
+    },
+    {
+      "epoch": 23.44,
+      "learning_rate": 3.111111111111112e-05,
+      "loss": 5.4921,
+      "step": 30000
+    },
+    {
+      "epoch": 24.22,
+      "learning_rate": 3.066666666666667e-05,
+      "loss": 5.4554,
+      "step": 31000
+    },
+    {
+      "epoch": 25.0,
+      "learning_rate": 3.0222222222222225e-05,
+      "loss": 5.4237,
+      "step": 32000
+    },
+    {
+      "epoch": 25.78,
+      "learning_rate": 2.977777777777778e-05,
+      "loss": 5.3938,
+      "step": 33000
+    },
+    {
+      "epoch": 26.56,
+      "learning_rate": 2.9333333333333333e-05,
+      "loss": 5.3701,
+      "step": 34000
+    },
+    {
+      "epoch": 27.34,
+      "learning_rate": 2.888888888888889e-05,
+      "loss": 5.3452,
+      "step": 35000
+    },
+    {
+      "epoch": 28.12,
+      "learning_rate": 2.8444444444444447e-05,
+      "loss": 5.327,
+      "step": 36000
+    },
+    {
+      "epoch": 28.91,
+      "learning_rate": 2.8e-05,
+      "loss": 5.3063,
+      "step": 37000
+    },
+    {
+      "epoch": 29.69,
+      "learning_rate": 2.755555555555556e-05,
+      "loss": 5.2905,
+      "step": 38000
+    },
+    {
+      "epoch": 30.47,
+      "learning_rate": 2.7111111111111114e-05,
+      "loss": 5.2747,
+      "step": 39000
+    },
+    {
+      "epoch": 31.25,
+      "learning_rate": 2.6666666666666667e-05,
+      "loss": 5.2596,
+      "step": 40000
+    },
+    {
+      "epoch": 32.03,
+      "learning_rate": 2.6222222222222226e-05,
+      "loss": 5.2483,
+      "step": 41000
+    },
+    {
+      "epoch": 32.81,
+      "learning_rate": 2.577777777777778e-05,
+      "loss": 5.2336,
+      "step": 42000
+    },
+    {
+      "epoch": 33.59,
+      "learning_rate": 2.5333333333333334e-05,
+      "loss": 5.2214,
+      "step": 43000
+    },
+    {
+      "epoch": 34.38,
+      "learning_rate": 2.4888888888888893e-05,
+      "loss": 5.2103,
+      "step": 44000
+    },
+    {
+      "epoch": 35.16,
+      "learning_rate": 2.444444444444445e-05,
+      "loss": 5.2015,
+      "step": 45000
+    },
+    {
+      "epoch": 35.94,
+      "learning_rate": 2.4e-05,
+      "loss": 5.1907,
+      "step": 46000
+    },
+    {
+      "epoch": 36.72,
+      "learning_rate": 2.355555555555556e-05,
+      "loss": 5.1801,
+      "step": 47000
+    },
+    {
+      "epoch": 37.5,
+      "learning_rate": 2.3111111111111112e-05,
+      "loss": 5.1713,
+      "step": 48000
+    },
+    {
+      "epoch": 38.28,
+      "learning_rate": 2.2666666666666668e-05,
+      "loss": 5.1645,
+      "step": 49000
+    },
+    {
+      "epoch": 39.06,
+      "learning_rate": 2.2222222222222227e-05,
+      "loss": 5.1548,
+      "step": 50000
+    },
+    {
+      "epoch": 39.84,
+      "learning_rate": 2.177777777777778e-05,
+      "loss": 5.1462,
+      "step": 51000
+    },
+    {
+      "epoch": 40.62,
+      "learning_rate": 2.1333333333333335e-05,
+      "loss": 5.1417,
+      "step": 52000
+    },
+    {
+      "epoch": 41.41,
+      "learning_rate": 2.088888888888889e-05,
+      "loss": 5.1305,
+      "step": 53000
+    },
+    {
+      "epoch": 42.19,
+      "learning_rate": 2.0444444444444446e-05,
+      "loss": 5.1287,
+      "step": 54000
+    },
+    {
+      "epoch": 42.97,
+      "learning_rate": 2e-05,
+      "loss": 5.1202,
+      "step": 55000
+    },
+    {
+      "epoch": 43.75,
+      "learning_rate": 1.9555555555555557e-05,
+      "loss": 5.1139,
+      "step": 56000
+    },
+    {
+      "epoch": 44.53,
+      "learning_rate": 1.9111111111111113e-05,
+      "loss": 5.1081,
+      "step": 57000
+    },
+    {
+      "epoch": 45.31,
+      "learning_rate": 1.866666666666667e-05,
+      "loss": 5.1032,
+      "step": 58000
+    },
+    {
+      "epoch": 46.09,
+      "learning_rate": 1.8222222222222224e-05,
+      "loss": 5.0968,
+      "step": 59000
+    },
+    {
+      "epoch": 46.88,
+      "learning_rate": 1.7777777777777777e-05,
+      "loss": 5.0914,
+      "step": 60000
+    },
+    {
+      "epoch": 47.66,
+      "learning_rate": 1.7333333333333336e-05,
+      "loss": 5.085,
+      "step": 61000
+    },
+    {
+      "epoch": 48.44,
+      "learning_rate": 1.688888888888889e-05,
+      "loss": 5.0806,
+      "step": 62000
+    },
+    {
+      "epoch": 49.22,
+      "learning_rate": 1.6444444444444444e-05,
+      "loss": 5.0769,
+      "step": 63000
+    },
+    {
+      "epoch": 50.0,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 5.0728,
+      "step": 64000
+    },
+    {
+      "epoch": 50.78,
+      "learning_rate": 1.555555555555556e-05,
+      "loss": 5.0692,
+      "step": 65000
+    },
+    {
+      "epoch": 51.56,
+      "learning_rate": 1.5111111111111112e-05,
+      "loss": 5.065,
+      "step": 66000
+    },
+    {
+      "epoch": 52.34,
+      "learning_rate": 1.4666666666666666e-05,
+      "loss": 5.0594,
+      "step": 67000
+    },
+    {
+      "epoch": 53.12,
+      "learning_rate": 1.4222222222222224e-05,
+      "loss": 5.0549,
+      "step": 68000
+    },
+    {
+      "epoch": 53.91,
+      "learning_rate": 1.377777777777778e-05,
+      "loss": 5.0538,
+      "step": 69000
+    },
+    {
+      "epoch": 54.69,
+      "learning_rate": 1.3333333333333333e-05,
+      "loss": 5.0482,
+      "step": 70000
+    },
+    {
+      "epoch": 55.47,
+      "learning_rate": 1.288888888888889e-05,
+      "loss": 5.0475,
+      "step": 71000
+    },
+    {
+      "epoch": 56.25,
+      "learning_rate": 1.2444444444444446e-05,
+      "loss": 5.0399,
+      "step": 72000
+    },
+    {
+      "epoch": 57.03,
+      "learning_rate": 1.2e-05,
+      "loss": 5.0406,
+      "step": 73000
+    },
+    {
+      "epoch": 57.81,
+      "learning_rate": 1.1555555555555556e-05,
+      "loss": 5.037,
+      "step": 74000
+    },
+    {
+      "epoch": 58.59,
+      "learning_rate": 1.1111111111111113e-05,
+      "loss": 5.0326,
+      "step": 75000
+    },
+    {
+      "epoch": 59.38,
+      "learning_rate": 1.0666666666666667e-05,
+      "loss": 5.031,
+      "step": 76000
+    },
+    {
+      "epoch": 60.16,
+      "learning_rate": 1.0222222222222223e-05,
+      "loss": 5.0291,
+      "step": 77000
+    },
+    {
+      "epoch": 60.94,
+      "learning_rate": 9.777777777777779e-06,
+      "loss": 5.0266,
+      "step": 78000
+    },
+    {
+      "epoch": 61.72,
+      "learning_rate": 9.333333333333334e-06,
+      "loss": 5.0235,
+      "step": 79000
+    },
+    {
+      "epoch": 62.5,
+      "learning_rate": 8.888888888888888e-06,
+      "loss": 5.0211,
+      "step": 80000
+    },
+    {
+      "epoch": 63.28,
+      "learning_rate": 8.444444444444446e-06,
+      "loss": 5.0214,
+      "step": 81000
+    },
+    {
+      "epoch": 64.06,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 5.0176,
+      "step": 82000
+    },
+    {
+      "epoch": 64.84,
+      "learning_rate": 7.555555555555556e-06,
+      "loss": 5.0157,
+      "step": 83000
+    },
+    {
+      "epoch": 65.62,
+      "learning_rate": 7.111111111111112e-06,
+      "loss": 5.014,
+      "step": 84000
+    },
+    {
+      "epoch": 66.41,
+      "learning_rate": 6.666666666666667e-06,
+      "loss": 5.0115,
+      "step": 85000
+    },
+    {
+      "epoch": 67.19,
+      "learning_rate": 6.222222222222223e-06,
+      "loss": 5.0087,
+      "step": 86000
+    },
+    {
+      "epoch": 67.97,
+      "learning_rate": 5.777777777777778e-06,
+      "loss": 5.0105,
+      "step": 87000
+    },
+    {
+      "epoch": 68.75,
+      "learning_rate": 5.333333333333334e-06,
+      "loss": 5.0062,
+      "step": 88000
+    },
+    {
+      "epoch": 69.53,
+      "learning_rate": 4.888888888888889e-06,
+      "loss": 5.0028,
+      "step": 89000
+    },
+    {
+      "epoch": 70.31,
+      "learning_rate": 4.444444444444444e-06,
+      "loss": 5.0066,
+      "step": 90000
+    },
+    {
+      "epoch": 71.09,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 5.0032,
+      "step": 91000
+    },
+    {
+      "epoch": 71.88,
+      "learning_rate": 3.555555555555556e-06,
+      "loss": 5.0022,
+      "step": 92000
+    },
+    {
+      "epoch": 72.66,
+      "learning_rate": 3.1111111111111116e-06,
+      "loss": 5.0024,
+      "step": 93000
+    },
+    {
+      "epoch": 73.44,
+      "learning_rate": 2.666666666666667e-06,
+      "loss": 5.0007,
+      "step": 94000
+    },
+    {
+      "epoch": 74.22,
+      "learning_rate": 2.222222222222222e-06,
+      "loss": 5.0008,
+      "step": 95000
+    },
+    {
+      "epoch": 75.0,
+      "learning_rate": 1.777777777777778e-06,
+      "loss": 4.9978,
+      "step": 96000
+    },
+    {
+      "epoch": 75.78,
+      "learning_rate": 1.3333333333333334e-06,
+      "loss": 4.9966,
+      "step": 97000
+    },
+    {
+      "epoch": 76.56,
+      "learning_rate": 8.88888888888889e-07,
+      "loss": 4.9994,
+      "step": 98000
+    },
+    {
+      "epoch": 77.34,
+      "learning_rate": 4.444444444444445e-07,
+      "loss": 4.9966,
+      "step": 99000
+    },
+    {
+      "epoch": 78.12,
+      "learning_rate": 0.0,
+      "loss": 4.9976,
+      "step": 100000
+    },
+    {
+      "epoch": 9.87,
+      "learning_rate": 2.0842105263157895e-05,
+      "loss": 5.0155,
+      "step": 101000
+    },
+    {
+      "epoch": 9.97,
+      "learning_rate": 2.0631578947368425e-05,
+      "loss": 5.024,
+      "step": 102000
+    },
+    {
+      "epoch": 10.06,
+      "learning_rate": 2.0421052631578952e-05,
+      "loss": 5.0241,
+      "step": 103000
+    },
+    {
+      "epoch": 10.16,
+      "learning_rate": 2.0210526315789475e-05,
+      "loss": 5.0218,
+      "step": 104000
+    },
+    {
+      "epoch": 10.26,
+      "learning_rate": 2e-05,
+      "loss": 5.0231,
+      "step": 105000
+    },
+    {
+      "epoch": 10.36,
+      "learning_rate": 1.9789473684210528e-05,
+      "loss": 5.0231,
+      "step": 106000
+    },
+    {
+      "epoch": 10.45,
+      "learning_rate": 1.9578947368421055e-05,
+      "loss": 5.0198,
+      "step": 107000
+    },
+    {
+      "epoch": 10.55,
+      "learning_rate": 1.936842105263158e-05,
+      "loss": 5.0158,
+      "step": 108000
+    },
+    {
+      "epoch": 10.65,
+      "learning_rate": 1.9157894736842108e-05,
+      "loss": 5.0122,
+      "step": 109000
+    },
+    {
+      "epoch": 10.75,
+      "learning_rate": 1.894736842105263e-05,
+      "loss": 5.0165,
+      "step": 110000
+    },
+    {
+      "epoch": 10.85,
+      "learning_rate": 1.873684210526316e-05,
+      "loss": 5.0183,
+      "step": 111000
+    },
+    {
+      "epoch": 10.94,
+      "learning_rate": 1.8526315789473684e-05,
+      "loss": 5.01,
+      "step": 112000
+    },
+    {
+      "epoch": 11.04,
+      "learning_rate": 1.831578947368421e-05,
+      "loss": 5.007,
+      "step": 113000
+    },
+    {
+      "epoch": 11.14,
+      "learning_rate": 1.810526315789474e-05,
+      "loss": 5.0059,
+      "step": 114000
+    },
+    {
+      "epoch": 11.24,
+      "learning_rate": 1.7894736842105264e-05,
+      "loss": 5.0089,
+      "step": 115000
+    },
+    {
+      "epoch": 11.33,
+      "learning_rate": 1.768421052631579e-05,
+      "loss": 5.0081,
+      "step": 116000
+    },
+    {
+      "epoch": 11.43,
+      "learning_rate": 1.7473684210526317e-05,
+      "loss": 5.002,
+      "step": 117000
+    },
+    {
+      "epoch": 11.53,
+      "learning_rate": 1.7263157894736843e-05,
+      "loss": 5.0052,
+      "step": 118000
+    },
+    {
+      "epoch": 11.63,
+      "learning_rate": 1.705263157894737e-05,
+      "loss": 5.0079,
+      "step": 119000
+    },
+    {
+      "epoch": 11.72,
+      "learning_rate": 1.6842105263157896e-05,
+      "loss": 4.9985,
+      "step": 120000
+    },
+    {
+      "epoch": 11.82,
+      "learning_rate": 1.6631578947368423e-05,
+      "loss": 5.006,
+      "step": 121000
+    },
+    {
+      "epoch": 11.92,
+      "learning_rate": 1.642105263157895e-05,
+      "loss": 4.9956,
+      "step": 122000
+    },
+    {
+      "epoch": 12.02,
+      "learning_rate": 1.6210526315789473e-05,
+      "loss": 4.9973,
+      "step": 123000
+    },
+    {
+      "epoch": 12.12,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 4.9976,
+      "step": 124000
+    },
+    {
+      "epoch": 12.21,
+      "learning_rate": 1.578947368421053e-05,
+      "loss": 4.9882,
+      "step": 125000
+    },
+    {
+      "epoch": 12.31,
+      "learning_rate": 1.5578947368421052e-05,
+      "loss": 4.9908,
+      "step": 126000
+    },
+    {
+      "epoch": 12.41,
+      "learning_rate": 1.536842105263158e-05,
+      "loss": 5.0038,
+      "step": 127000
+    },
+    {
+      "epoch": 12.51,
+      "learning_rate": 1.5157894736842107e-05,
+      "loss": 4.9923,
+      "step": 128000
+    },
+    {
+      "epoch": 12.6,
+      "learning_rate": 1.4947368421052632e-05,
+      "loss": 4.9847,
+      "step": 129000
+    },
+    {
+      "epoch": 12.7,
+      "learning_rate": 1.4736842105263159e-05,
+      "loss": 4.9864,
+      "step": 130000
+    },
+    {
+      "epoch": 12.8,
+      "learning_rate": 1.4526315789473687e-05,
+      "loss": 4.9903,
+      "step": 131000
+    },
+    {
+      "epoch": 12.9,
+      "learning_rate": 1.4315789473684212e-05,
+      "loss": 4.9812,
+      "step": 132000
+    },
+    {
+      "epoch": 12.99,
+      "learning_rate": 1.4105263157894738e-05,
+      "loss": 4.9923,
+      "step": 133000
+    },
+    {
+      "epoch": 13.09,
+      "learning_rate": 1.3894736842105265e-05,
+      "loss": 4.9838,
+      "step": 134000
+    },
+    {
+      "epoch": 13.19,
+      "learning_rate": 1.3684210526315791e-05,
+      "loss": 4.977,
+      "step": 135000
+    },
+    {
+      "epoch": 13.29,
+      "learning_rate": 1.3473684210526316e-05,
+      "loss": 4.9743,
+      "step": 136000
+    },
+    {
+      "epoch": 13.39,
+      "learning_rate": 1.3263157894736843e-05,
+      "loss": 4.9797,
+      "step": 137000
+    },
+    {
+      "epoch": 13.48,
+      "learning_rate": 1.305263157894737e-05,
+      "loss": 4.9915,
+      "step": 138000
+    },
+    {
+      "epoch": 13.58,
+      "learning_rate": 1.2842105263157896e-05,
+      "loss": 4.9762,
+      "step": 139000
+    },
+    {
+      "epoch": 13.68,
+      "learning_rate": 1.263157894736842e-05,
+      "loss": 4.9771,
+      "step": 140000
+    },
+    {
+      "epoch": 13.78,
+      "learning_rate": 1.2421052631578949e-05,
+      "loss": 4.9763,
+      "step": 141000
+    },
+    {
+      "epoch": 13.87,
+      "learning_rate": 1.2210526315789475e-05,
+      "loss": 4.9706,
+      "step": 142000
+    },
+    {
+      "epoch": 13.97,
+      "learning_rate": 1.2e-05,
+      "loss": 4.9731,
+      "step": 143000
+    },
+    {
+      "epoch": 14.07,
+      "learning_rate": 1.1789473684210527e-05,
+      "loss": 4.9736,
+      "step": 144000
+    },
+    {
+      "epoch": 14.17,
+      "learning_rate": 1.1578947368421053e-05,
+      "loss": 4.9704,
+      "step": 145000
+    },
+    {
+      "epoch": 14.26,
+      "learning_rate": 1.136842105263158e-05,
+      "loss": 4.9762,
+      "step": 146000
+    },
+    {
+      "epoch": 14.36,
+      "learning_rate": 1.1157894736842105e-05,
+      "loss": 4.9709,
+      "step": 147000
+    },
+    {
+      "epoch": 14.46,
+      "learning_rate": 1.0947368421052633e-05,
+      "loss": 4.957,
+      "step": 148000
+    },
+    {
+      "epoch": 14.56,
+      "learning_rate": 1.073684210526316e-05,
+      "loss": 4.9666,
+      "step": 149000
+    },
+    {
+      "epoch": 14.66,
+      "learning_rate": 1.0526315789473684e-05,
+      "loss": 4.9623,
+      "step": 150000
+    },
+    {
+      "epoch": 14.75,
+      "learning_rate": 1.0315789473684213e-05,
+      "loss": 4.9613,
+      "step": 151000
+    },
+    {
+      "epoch": 14.85,
+      "learning_rate": 1.0105263157894738e-05,
+      "loss": 4.9642,
+      "step": 152000
+    },
+    {
+      "epoch": 14.95,
+      "learning_rate": 9.894736842105264e-06,
+      "loss": 4.9605,
+      "step": 153000
+    },
+    {
+      "epoch": 15.05,
+      "learning_rate": 9.68421052631579e-06,
+      "loss": 4.9688,
+      "step": 154000
+    },
+    {
+      "epoch": 15.14,
+      "learning_rate": 9.473684210526315e-06,
+      "loss": 4.9635,
+      "step": 155000
+    },
+    {
+      "epoch": 15.24,
+      "learning_rate": 9.263157894736842e-06,
+      "loss": 4.9587,
+      "step": 156000
+    },
+    {
+      "epoch": 15.34,
+      "learning_rate": 9.05263157894737e-06,
+      "loss": 4.9633,
+      "step": 157000
+    },
+    {
+      "epoch": 15.44,
+      "learning_rate": 8.842105263157895e-06,
+      "loss": 4.9613,
+      "step": 158000
+    },
+    {
+      "epoch": 15.53,
+      "learning_rate": 8.631578947368422e-06,
+      "loss": 4.9629,
+      "step": 159000
+    },
+    {
+      "epoch": 15.63,
+      "learning_rate": 8.421052631578948e-06,
+      "loss": 4.9547,
+      "step": 160000
+    },
+    {
+      "epoch": 15.73,
+      "learning_rate": 8.210526315789475e-06,
+      "loss": 4.9541,
+      "step": 161000
+    },
+    {
+      "epoch": 15.83,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 4.9532,
+      "step": 162000
+    },
+    {
+      "epoch": 15.93,
+      "learning_rate": 7.789473684210526e-06,
+      "loss": 4.9461,
+      "step": 163000
+    },
+    {
+      "epoch": 16.02,
+      "learning_rate": 7.578947368421054e-06,
+      "loss": 4.9518,
+      "step": 164000
+    },
+    {
+      "epoch": 16.12,
+      "learning_rate": 7.368421052631579e-06,
+      "loss": 4.9543,
+      "step": 165000
+    },
+    {
+      "epoch": 16.22,
+      "learning_rate": 7.157894736842106e-06,
+      "loss": 4.9526,
+      "step": 166000
+    },
+    {
+      "epoch": 16.32,
+      "learning_rate": 6.947368421052632e-06,
+      "loss": 4.9447,
+      "step": 167000
+    },
+    {
+      "epoch": 16.41,
+      "learning_rate": 6.736842105263158e-06,
+      "loss": 4.9545,
+      "step": 168000
+    },
+    {
+      "epoch": 16.51,
+      "learning_rate": 6.526315789473685e-06,
+      "loss": 4.9448,
+      "step": 169000
+    },
+    {
+      "epoch": 16.61,
+      "learning_rate": 6.31578947368421e-06,
+      "loss": 4.9521,
+      "step": 170000
+    },
+    {
+      "epoch": 16.71,
+      "learning_rate": 6.105263157894738e-06,
+      "loss": 4.9503,
+      "step": 171000
+    },
+    {
+      "epoch": 16.81,
+      "learning_rate": 5.8947368421052634e-06,
+      "loss": 4.9537,
+      "step": 172000
+    },
+    {
+      "epoch": 16.9,
+      "learning_rate": 5.68421052631579e-06,
+      "loss": 4.9446,
+      "step": 173000
+    },
+    {
+      "epoch": 17.0,
+      "learning_rate": 5.4736842105263165e-06,
+      "loss": 4.9397,
+      "step": 174000
+    },
+    {
+      "epoch": 17.1,
+      "learning_rate": 5.263157894736842e-06,
+      "loss": 4.9541,
+      "step": 175000
+    },
+    {
+      "epoch": 17.2,
+      "learning_rate": 5.052631578947369e-06,
+      "loss": 4.9449,
+      "step": 176000
+    },
+    {
+      "epoch": 17.29,
+      "learning_rate": 4.842105263157895e-06,
+      "loss": 4.9447,
+      "step": 177000
+    },
+    {
+      "epoch": 17.39,
+      "learning_rate": 4.631578947368421e-06,
+      "loss": 4.9344,
+      "step": 178000
+    },
+    {
+      "epoch": 17.49,
+      "learning_rate": 4.4210526315789476e-06,
+      "loss": 4.933,
+      "step": 179000
+    },
+    {
+      "epoch": 17.59,
+      "learning_rate": 4.210526315789474e-06,
+      "loss": 4.9397,
+      "step": 180000
+    },
+    {
+      "epoch": 17.68,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 4.9431,
+      "step": 181000
+    },
+    {
+      "epoch": 17.78,
+      "learning_rate": 3.789473684210527e-06,
+      "loss": 4.9378,
+      "step": 182000
+    },
+    {
+      "epoch": 17.88,
+      "learning_rate": 3.578947368421053e-06,
+      "loss": 4.9399,
+      "step": 183000
+    },
+    {
+      "epoch": 17.98,
+      "learning_rate": 3.368421052631579e-06,
+      "loss": 4.9481,
+      "step": 184000
+    },
+    {
+      "epoch": 18.08,
+      "learning_rate": 3.157894736842105e-06,
+      "loss": 4.9399,
+      "step": 185000
+    },
+    {
+      "epoch": 18.17,
+      "learning_rate": 2.9473684210526317e-06,
+      "loss": 4.9365,
+      "step": 186000
+    },
+    {
+      "epoch": 18.27,
+      "learning_rate": 2.7368421052631583e-06,
+      "loss": 4.9307,
+      "step": 187000
+    },
+    {
+      "epoch": 18.37,
+      "learning_rate": 2.5263157894736844e-06,
+      "loss": 4.9391,
+      "step": 188000
+    },
+    {
+      "epoch": 18.47,
+      "learning_rate": 2.3157894736842105e-06,
+      "loss": 4.9331,
+      "step": 189000
+    },
+    {
+      "epoch": 18.56,
+      "learning_rate": 2.105263157894737e-06,
+      "loss": 4.948,
+      "step": 190000
+    },
+    {
+      "epoch": 18.66,
+      "learning_rate": 1.8947368421052634e-06,
+      "loss": 4.9407,
+      "step": 191000
+    },
+    {
+      "epoch": 18.76,
+      "learning_rate": 1.6842105263157895e-06,
+      "loss": 4.944,
+      "step": 192000
+    },
+    {
+      "epoch": 18.86,
+      "learning_rate": 1.4736842105263159e-06,
+      "loss": 4.9286,
+      "step": 193000
+    }
+  ],
+  "logging_steps": 1000,
+  "max_steps": 200000,
+  "num_train_epochs": 20,
+  "save_steps": 1000,
+  "total_flos": 2.2557198125934313e+19,
+  "trial_name": null,
+  "trial_params": null
+}

len2_5120/checkpoint-193000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9ff5d6388a59708cb6305a7fab984d438e385b1835680f9629b90c23c5cf704
+size 5137

len2_5120/checkpoint-194000/config.json

@@ -0,0 +1,25 @@
+{
+  "architectures": [
+    "BertForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_length": 512,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.35.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 5120
+}

len2_5120/checkpoint-194000/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "max_length": 512,
+  "pad_token_id": 0,
+  "transformers_version": "4.35.2"
+}

len2_5120/checkpoint-194000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4dddd1badead3e61bee03b327cb7238008ee87237c32a76145e5d7b8cd63c311
+size 359944256

len2_5120/checkpoint-194000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bfbb972435c818c69a8a43aae08e74deb9a6222aede1f026af4eefdd90269e25
+size 720010699

len2_5120/checkpoint-194000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6960f2857e1f35b7d4b58eb747b95faa2f97f460296f4fb9eeb9499e4792b82e
+size 14645

len2_5120/checkpoint-194000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9aa460464675e9f0593bb3962bdd48c1f52b25c8c8bf257806850efc7689aa1
+size 1465

len2_5120/checkpoint-194000/trainer_state.json

@@ -0,0 +1,1183 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 18.95456765999023,
+  "eval_steps": 500,
+  "global_step": 194000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.78,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 7.5832,
+      "step": 1000
+    },
+    {
+      "epoch": 1.56,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 6.9724,
+      "step": 2000
+    },
+    {
+      "epoch": 2.34,
+      "learning_rate": 1.2e-05,
+      "loss": 6.7695,
+      "step": 3000
+    },
+    {
+      "epoch": 3.12,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 6.7002,
+      "step": 4000
+    },
+    {
+      "epoch": 3.91,
+      "learning_rate": 2e-05,
+      "loss": 6.666,
+      "step": 5000
+    },
+    {
+      "epoch": 4.69,
+      "learning_rate": 2.4e-05,
+      "loss": 6.6503,
+      "step": 6000
+    },
+    {
+      "epoch": 5.47,
+      "learning_rate": 2.8e-05,
+      "loss": 6.6407,
+      "step": 7000
+    },
+    {
+      "epoch": 6.25,
+      "learning_rate": 3.2000000000000005e-05,
+      "loss": 6.6277,
+      "step": 8000
+    },
+    {
+      "epoch": 7.03,
+      "learning_rate": 3.6e-05,
+      "loss": 6.6209,
+      "step": 9000
+    },
+    {
+      "epoch": 7.81,
+      "learning_rate": 4e-05,
+      "loss": 6.6135,
+      "step": 10000
+    },
+    {
+      "epoch": 8.59,
+      "learning_rate": 3.955555555555556e-05,
+      "loss": 6.6084,
+      "step": 11000
+    },
+    {
+      "epoch": 9.38,
+      "learning_rate": 3.9111111111111115e-05,
+      "loss": 6.6041,
+      "step": 12000
+    },
+    {
+      "epoch": 10.16,
+      "learning_rate": 3.866666666666667e-05,
+      "loss": 6.6013,
+      "step": 13000
+    },
+    {
+      "epoch": 10.94,
+      "learning_rate": 3.8222222222222226e-05,
+      "loss": 6.5994,
+      "step": 14000
+    },
+    {
+      "epoch": 11.72,
+      "learning_rate": 3.777777777777778e-05,
+      "loss": 6.594,
+      "step": 15000
+    },
+    {
+      "epoch": 12.5,
+      "learning_rate": 3.733333333333334e-05,
+      "loss": 6.5925,
+      "step": 16000
+    },
+    {
+      "epoch": 13.28,
+      "learning_rate": 3.6888888888888896e-05,
+      "loss": 6.5893,
+      "step": 17000
+    },
+    {
+      "epoch": 14.06,
+      "learning_rate": 3.644444444444445e-05,
+      "loss": 6.5875,
+      "step": 18000
+    },
+    {
+      "epoch": 14.84,
+      "learning_rate": 3.6e-05,
+      "loss": 6.5232,
+      "step": 19000
+    },
+    {
+      "epoch": 15.62,
+      "learning_rate": 3.555555555555555e-05,
+      "loss": 6.4358,
+      "step": 20000
+    },
+    {
+      "epoch": 16.41,
+      "learning_rate": 3.511111111111111e-05,
+      "loss": 6.3712,
+      "step": 21000
+    },
+    {
+      "epoch": 17.19,
+      "learning_rate": 3.466666666666667e-05,
+      "loss": 6.311,
+      "step": 22000
+    },
+    {
+      "epoch": 17.97,
+      "learning_rate": 3.4222222222222224e-05,
+      "loss": 6.2493,
+      "step": 23000
+    },
+    {
+      "epoch": 18.75,
+      "learning_rate": 3.377777777777778e-05,
+      "loss": 6.1597,
+      "step": 24000
+    },
+    {
+      "epoch": 19.53,
+      "learning_rate": 3.3333333333333335e-05,
+      "loss": 6.0276,
+      "step": 25000
+    },
+    {
+      "epoch": 20.31,
+      "learning_rate": 3.288888888888889e-05,
+      "loss": 5.8317,
+      "step": 26000
+    },
+    {
+      "epoch": 21.09,
+      "learning_rate": 3.2444444444444446e-05,
+      "loss": 5.6782,
+      "step": 27000
+    },
+    {
+      "epoch": 21.88,
+      "learning_rate": 3.2000000000000005e-05,
+      "loss": 5.5975,
+      "step": 28000
+    },
+    {
+      "epoch": 22.66,
+      "learning_rate": 3.155555555555556e-05,
+      "loss": 5.5369,
+      "step": 29000
+    },
+    {
+      "epoch": 23.44,
+      "learning_rate": 3.111111111111112e-05,
+      "loss": 5.4921,
+      "step": 30000
+    },
+    {
+      "epoch": 24.22,
+      "learning_rate": 3.066666666666667e-05,
+      "loss": 5.4554,
+      "step": 31000
+    },
+    {
+      "epoch": 25.0,
+      "learning_rate": 3.0222222222222225e-05,
+      "loss": 5.4237,
+      "step": 32000
+    },
+    {
+      "epoch": 25.78,
+      "learning_rate": 2.977777777777778e-05,
+      "loss": 5.3938,
+      "step": 33000
+    },
+    {
+      "epoch": 26.56,
+      "learning_rate": 2.9333333333333333e-05,
+      "loss": 5.3701,
+      "step": 34000
+    },
+    {
+      "epoch": 27.34,
+      "learning_rate": 2.888888888888889e-05,
+      "loss": 5.3452,
+      "step": 35000
+    },
+    {
+      "epoch": 28.12,
+      "learning_rate": 2.8444444444444447e-05,
+      "loss": 5.327,
+      "step": 36000
+    },
+    {
+      "epoch": 28.91,
+      "learning_rate": 2.8e-05,
+      "loss": 5.3063,
+      "step": 37000
+    },
+    {
+      "epoch": 29.69,
+      "learning_rate": 2.755555555555556e-05,
+      "loss": 5.2905,
+      "step": 38000
+    },
+    {
+      "epoch": 30.47,
+      "learning_rate": 2.7111111111111114e-05,
+      "loss": 5.2747,
+      "step": 39000
+    },
+    {
+      "epoch": 31.25,
+      "learning_rate": 2.6666666666666667e-05,
+      "loss": 5.2596,
+      "step": 40000
+    },
+    {
+      "epoch": 32.03,
+      "learning_rate": 2.6222222222222226e-05,
+      "loss": 5.2483,
+      "step": 41000
+    },
+    {
+      "epoch": 32.81,
+      "learning_rate": 2.577777777777778e-05,
+      "loss": 5.2336,
+      "step": 42000
+    },
+    {
+      "epoch": 33.59,
+      "learning_rate": 2.5333333333333334e-05,
+      "loss": 5.2214,
+      "step": 43000
+    },
+    {
+      "epoch": 34.38,
+      "learning_rate": 2.4888888888888893e-05,
+      "loss": 5.2103,
+      "step": 44000
+    },
+    {
+      "epoch": 35.16,
+      "learning_rate": 2.444444444444445e-05,
+      "loss": 5.2015,
+      "step": 45000
+    },
+    {
+      "epoch": 35.94,
+      "learning_rate": 2.4e-05,
+      "loss": 5.1907,
+      "step": 46000
+    },
+    {
+      "epoch": 36.72,
+      "learning_rate": 2.355555555555556e-05,
+      "loss": 5.1801,
+      "step": 47000
+    },
+    {
+      "epoch": 37.5,
+      "learning_rate": 2.3111111111111112e-05,
+      "loss": 5.1713,
+      "step": 48000
+    },
+    {
+      "epoch": 38.28,
+      "learning_rate": 2.2666666666666668e-05,
+      "loss": 5.1645,
+      "step": 49000
+    },
+    {
+      "epoch": 39.06,
+      "learning_rate": 2.2222222222222227e-05,
+      "loss": 5.1548,
+      "step": 50000
+    },
+    {
+      "epoch": 39.84,
+      "learning_rate": 2.177777777777778e-05,
+      "loss": 5.1462,
+      "step": 51000
+    },
+    {
+      "epoch": 40.62,
+      "learning_rate": 2.1333333333333335e-05,
+      "loss": 5.1417,
+      "step": 52000
+    },
+    {
+      "epoch": 41.41,
+      "learning_rate": 2.088888888888889e-05,
+      "loss": 5.1305,
+      "step": 53000
+    },
+    {
+      "epoch": 42.19,
+      "learning_rate": 2.0444444444444446e-05,
+      "loss": 5.1287,
+      "step": 54000
+    },
+    {
+      "epoch": 42.97,
+      "learning_rate": 2e-05,
+      "loss": 5.1202,
+      "step": 55000
+    },
+    {
+      "epoch": 43.75,
+      "learning_rate": 1.9555555555555557e-05,
+      "loss": 5.1139,
+      "step": 56000
+    },
+    {
+      "epoch": 44.53,
+      "learning_rate": 1.9111111111111113e-05,
+      "loss": 5.1081,
+      "step": 57000
+    },
+    {
+      "epoch": 45.31,
+      "learning_rate": 1.866666666666667e-05,
+      "loss": 5.1032,
+      "step": 58000
+    },
+    {
+      "epoch": 46.09,
+      "learning_rate": 1.8222222222222224e-05,
+      "loss": 5.0968,
+      "step": 59000
+    },
+    {
+      "epoch": 46.88,
+      "learning_rate": 1.7777777777777777e-05,
+      "loss": 5.0914,
+      "step": 60000
+    },
+    {
+      "epoch": 47.66,
+      "learning_rate": 1.7333333333333336e-05,
+      "loss": 5.085,
+      "step": 61000
+    },
+    {
+      "epoch": 48.44,
+      "learning_rate": 1.688888888888889e-05,
+      "loss": 5.0806,
+      "step": 62000
+    },
+    {
+      "epoch": 49.22,
+      "learning_rate": 1.6444444444444444e-05,
+      "loss": 5.0769,
+      "step": 63000
+    },
+    {
+      "epoch": 50.0,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 5.0728,
+      "step": 64000
+    },
+    {
+      "epoch": 50.78,
+      "learning_rate": 1.555555555555556e-05,
+      "loss": 5.0692,
+      "step": 65000
+    },
+    {
+      "epoch": 51.56,
+      "learning_rate": 1.5111111111111112e-05,
+      "loss": 5.065,
+      "step": 66000
+    },
+    {
+      "epoch": 52.34,
+      "learning_rate": 1.4666666666666666e-05,
+      "loss": 5.0594,
+      "step": 67000
+    },
+    {
+      "epoch": 53.12,
+      "learning_rate": 1.4222222222222224e-05,
+      "loss": 5.0549,
+      "step": 68000
+    },
+    {
+      "epoch": 53.91,
+      "learning_rate": 1.377777777777778e-05,
+      "loss": 5.0538,
+      "step": 69000
+    },
+    {
+      "epoch": 54.69,
+      "learning_rate": 1.3333333333333333e-05,
+      "loss": 5.0482,
+      "step": 70000
+    },
+    {
+      "epoch": 55.47,
+      "learning_rate": 1.288888888888889e-05,
+      "loss": 5.0475,
+      "step": 71000
+    },
+    {
+      "epoch": 56.25,
+      "learning_rate": 1.2444444444444446e-05,
+      "loss": 5.0399,
+      "step": 72000
+    },
+    {
+      "epoch": 57.03,
+      "learning_rate": 1.2e-05,
+      "loss": 5.0406,
+      "step": 73000
+    },
+    {
+      "epoch": 57.81,
+      "learning_rate": 1.1555555555555556e-05,
+      "loss": 5.037,
+      "step": 74000
+    },
+    {
+      "epoch": 58.59,
+      "learning_rate": 1.1111111111111113e-05,
+      "loss": 5.0326,
+      "step": 75000
+    },
+    {
+      "epoch": 59.38,
+      "learning_rate": 1.0666666666666667e-05,
+      "loss": 5.031,
+      "step": 76000
+    },
+    {
+      "epoch": 60.16,
+      "learning_rate": 1.0222222222222223e-05,
+      "loss": 5.0291,
+      "step": 77000
+    },
+    {
+      "epoch": 60.94,
+      "learning_rate": 9.777777777777779e-06,
+      "loss": 5.0266,
+      "step": 78000
+    },
+    {
+      "epoch": 61.72,
+      "learning_rate": 9.333333333333334e-06,
+      "loss": 5.0235,
+      "step": 79000
+    },
+    {
+      "epoch": 62.5,
+      "learning_rate": 8.888888888888888e-06,
+      "loss": 5.0211,
+      "step": 80000
+    },
+    {
+      "epoch": 63.28,
+      "learning_rate": 8.444444444444446e-06,
+      "loss": 5.0214,
+      "step": 81000
+    },
+    {
+      "epoch": 64.06,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 5.0176,
+      "step": 82000
+    },
+    {
+      "epoch": 64.84,
+      "learning_rate": 7.555555555555556e-06,
+      "loss": 5.0157,
+      "step": 83000
+    },
+    {
+      "epoch": 65.62,
+      "learning_rate": 7.111111111111112e-06,
+      "loss": 5.014,
+      "step": 84000
+    },
+    {
+      "epoch": 66.41,
+      "learning_rate": 6.666666666666667e-06,
+      "loss": 5.0115,
+      "step": 85000
+    },
+    {
+      "epoch": 67.19,
+      "learning_rate": 6.222222222222223e-06,
+      "loss": 5.0087,
+      "step": 86000
+    },
+    {
+      "epoch": 67.97,
+      "learning_rate": 5.777777777777778e-06,
+      "loss": 5.0105,
+      "step": 87000
+    },
+    {
+      "epoch": 68.75,
+      "learning_rate": 5.333333333333334e-06,
+      "loss": 5.0062,
+      "step": 88000
+    },
+    {
+      "epoch": 69.53,
+      "learning_rate": 4.888888888888889e-06,
+      "loss": 5.0028,
+      "step": 89000
+    },
+    {
+      "epoch": 70.31,
+      "learning_rate": 4.444444444444444e-06,
+      "loss": 5.0066,
+      "step": 90000
+    },
+    {
+      "epoch": 71.09,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 5.0032,
+      "step": 91000
+    },
+    {
+      "epoch": 71.88,
+      "learning_rate": 3.555555555555556e-06,
+      "loss": 5.0022,
+      "step": 92000
+    },
+    {
+      "epoch": 72.66,
+      "learning_rate": 3.1111111111111116e-06,
+      "loss": 5.0024,
+      "step": 93000
+    },
+    {
+      "epoch": 73.44,
+      "learning_rate": 2.666666666666667e-06,
+      "loss": 5.0007,
+      "step": 94000
+    },
+    {
+      "epoch": 74.22,
+      "learning_rate": 2.222222222222222e-06,
+      "loss": 5.0008,
+      "step": 95000
+    },
+    {
+      "epoch": 75.0,
+      "learning_rate": 1.777777777777778e-06,
+      "loss": 4.9978,
+      "step": 96000
+    },
+    {
+      "epoch": 75.78,
+      "learning_rate": 1.3333333333333334e-06,
+      "loss": 4.9966,
+      "step": 97000
+    },
+    {
+      "epoch": 76.56,
+      "learning_rate": 8.88888888888889e-07,
+      "loss": 4.9994,
+      "step": 98000
+    },
+    {
+      "epoch": 77.34,
+      "learning_rate": 4.444444444444445e-07,
+      "loss": 4.9966,
+      "step": 99000
+    },
+    {
+      "epoch": 78.12,
+      "learning_rate": 0.0,
+      "loss": 4.9976,
+      "step": 100000
+    },
+    {
+      "epoch": 9.87,
+      "learning_rate": 2.0842105263157895e-05,
+      "loss": 5.0155,
+      "step": 101000
+    },
+    {
+      "epoch": 9.97,
+      "learning_rate": 2.0631578947368425e-05,
+      "loss": 5.024,
+      "step": 102000
+    },
+    {
+      "epoch": 10.06,
+      "learning_rate": 2.0421052631578952e-05,
+      "loss": 5.0241,
+      "step": 103000
+    },
+    {
+      "epoch": 10.16,
+      "learning_rate": 2.0210526315789475e-05,
+      "loss": 5.0218,
+      "step": 104000
+    },
+    {
+      "epoch": 10.26,
+      "learning_rate": 2e-05,
+      "loss": 5.0231,
+      "step": 105000
+    },
+    {
+      "epoch": 10.36,
+      "learning_rate": 1.9789473684210528e-05,
+      "loss": 5.0231,
+      "step": 106000
+    },
+    {
+      "epoch": 10.45,
+      "learning_rate": 1.9578947368421055e-05,
+      "loss": 5.0198,
+      "step": 107000
+    },
+    {
+      "epoch": 10.55,
+      "learning_rate": 1.936842105263158e-05,
+      "loss": 5.0158,
+      "step": 108000
+    },
+    {
+      "epoch": 10.65,
+      "learning_rate": 1.9157894736842108e-05,
+      "loss": 5.0122,
+      "step": 109000
+    },
+    {
+      "epoch": 10.75,
+      "learning_rate": 1.894736842105263e-05,
+      "loss": 5.0165,
+      "step": 110000
+    },
+    {
+      "epoch": 10.85,
+      "learning_rate": 1.873684210526316e-05,
+      "loss": 5.0183,
+      "step": 111000
+    },
+    {
+      "epoch": 10.94,
+      "learning_rate": 1.8526315789473684e-05,
+      "loss": 5.01,
+      "step": 112000
+    },
+    {
+      "epoch": 11.04,
+      "learning_rate": 1.831578947368421e-05,
+      "loss": 5.007,
+      "step": 113000
+    },
+    {
+      "epoch": 11.14,
+      "learning_rate": 1.810526315789474e-05,
+      "loss": 5.0059,
+      "step": 114000
+    },
+    {
+      "epoch": 11.24,
+      "learning_rate": 1.7894736842105264e-05,
+      "loss": 5.0089,
+      "step": 115000
+    },
+    {
+      "epoch": 11.33,
+      "learning_rate": 1.768421052631579e-05,
+      "loss": 5.0081,
+      "step": 116000
+    },
+    {
+      "epoch": 11.43,
+      "learning_rate": 1.7473684210526317e-05,
+      "loss": 5.002,
+      "step": 117000
+    },
+    {
+      "epoch": 11.53,
+      "learning_rate": 1.7263157894736843e-05,
+      "loss": 5.0052,
+      "step": 118000
+    },
+    {
+      "epoch": 11.63,
+      "learning_rate": 1.705263157894737e-05,
+      "loss": 5.0079,
+      "step": 119000
+    },
+    {
+      "epoch": 11.72,
+      "learning_rate": 1.6842105263157896e-05,
+      "loss": 4.9985,
+      "step": 120000
+    },
+    {
+      "epoch": 11.82,
+      "learning_rate": 1.6631578947368423e-05,
+      "loss": 5.006,
+      "step": 121000
+    },
+    {
+      "epoch": 11.92,
+      "learning_rate": 1.642105263157895e-05,
+      "loss": 4.9956,
+      "step": 122000
+    },
+    {
+      "epoch": 12.02,
+      "learning_rate": 1.6210526315789473e-05,
+      "loss": 4.9973,
+      "step": 123000
+    },
+    {
+      "epoch": 12.12,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 4.9976,
+      "step": 124000
+    },
+    {
+      "epoch": 12.21,
+      "learning_rate": 1.578947368421053e-05,
+      "loss": 4.9882,
+      "step": 125000
+    },
+    {
+      "epoch": 12.31,
+      "learning_rate": 1.5578947368421052e-05,
+      "loss": 4.9908,
+      "step": 126000
+    },
+    {
+      "epoch": 12.41,
+      "learning_rate": 1.536842105263158e-05,
+      "loss": 5.0038,
+      "step": 127000
+    },
+    {
+      "epoch": 12.51,
+      "learning_rate": 1.5157894736842107e-05,
+      "loss": 4.9923,
+      "step": 128000
+    },
+    {
+      "epoch": 12.6,
+      "learning_rate": 1.4947368421052632e-05,
+      "loss": 4.9847,
+      "step": 129000
+    },
+    {
+      "epoch": 12.7,
+      "learning_rate": 1.4736842105263159e-05,
+      "loss": 4.9864,
+      "step": 130000
+    },
+    {
+      "epoch": 12.8,
+      "learning_rate": 1.4526315789473687e-05,
+      "loss": 4.9903,
+      "step": 131000
+    },
+    {
+      "epoch": 12.9,
+      "learning_rate": 1.4315789473684212e-05,
+      "loss": 4.9812,
+      "step": 132000
+    },
+    {
+      "epoch": 12.99,
+      "learning_rate": 1.4105263157894738e-05,
+      "loss": 4.9923,
+      "step": 133000
+    },
+    {
+      "epoch": 13.09,
+      "learning_rate": 1.3894736842105265e-05,
+      "loss": 4.9838,
+      "step": 134000
+    },
+    {
+      "epoch": 13.19,
+      "learning_rate": 1.3684210526315791e-05,
+      "loss": 4.977,
+      "step": 135000
+    },
+    {
+      "epoch": 13.29,
+      "learning_rate": 1.3473684210526316e-05,
+      "loss": 4.9743,
+      "step": 136000
+    },
+    {
+      "epoch": 13.39,
+      "learning_rate": 1.3263157894736843e-05,
+      "loss": 4.9797,
+      "step": 137000
+    },
+    {
+      "epoch": 13.48,
+      "learning_rate": 1.305263157894737e-05,
+      "loss": 4.9915,
+      "step": 138000
+    },
+    {
+      "epoch": 13.58,
+      "learning_rate": 1.2842105263157896e-05,
+      "loss": 4.9762,
+      "step": 139000
+    },
+    {
+      "epoch": 13.68,
+      "learning_rate": 1.263157894736842e-05,
+      "loss": 4.9771,
+      "step": 140000
+    },
+    {
+      "epoch": 13.78,
+      "learning_rate": 1.2421052631578949e-05,
+      "loss": 4.9763,
+      "step": 141000
+    },
+    {
+      "epoch": 13.87,
+      "learning_rate": 1.2210526315789475e-05,
+      "loss": 4.9706,
+      "step": 142000
+    },
+    {
+      "epoch": 13.97,
+      "learning_rate": 1.2e-05,
+      "loss": 4.9731,
+      "step": 143000
+    },
+    {
+      "epoch": 14.07,
+      "learning_rate": 1.1789473684210527e-05,
+      "loss": 4.9736,
+      "step": 144000
+    },
+    {
+      "epoch": 14.17,
+      "learning_rate": 1.1578947368421053e-05,
+      "loss": 4.9704,
+      "step": 145000
+    },
+    {
+      "epoch": 14.26,
+      "learning_rate": 1.136842105263158e-05,
+      "loss": 4.9762,
+      "step": 146000
+    },
+    {
+      "epoch": 14.36,
+      "learning_rate": 1.1157894736842105e-05,
+      "loss": 4.9709,
+      "step": 147000
+    },
+    {
+      "epoch": 14.46,
+      "learning_rate": 1.0947368421052633e-05,
+      "loss": 4.957,
+      "step": 148000
+    },
+    {
+      "epoch": 14.56,
+      "learning_rate": 1.073684210526316e-05,
+      "loss": 4.9666,
+      "step": 149000
+    },
+    {
+      "epoch": 14.66,
+      "learning_rate": 1.0526315789473684e-05,
+      "loss": 4.9623,
+      "step": 150000
+    },
+    {
+      "epoch": 14.75,
+      "learning_rate": 1.0315789473684213e-05,
+      "loss": 4.9613,
+      "step": 151000
+    },
+    {
+      "epoch": 14.85,
+      "learning_rate": 1.0105263157894738e-05,
+      "loss": 4.9642,
+      "step": 152000
+    },
+    {
+      "epoch": 14.95,
+      "learning_rate": 9.894736842105264e-06,
+      "loss": 4.9605,
+      "step": 153000
+    },
+    {
+      "epoch": 15.05,
+      "learning_rate": 9.68421052631579e-06,
+      "loss": 4.9688,
+      "step": 154000
+    },
+    {
+      "epoch": 15.14,
+      "learning_rate": 9.473684210526315e-06,
+      "loss": 4.9635,
+      "step": 155000
+    },
+    {
+      "epoch": 15.24,
+      "learning_rate": 9.263157894736842e-06,
+      "loss": 4.9587,
+      "step": 156000
+    },
+    {
+      "epoch": 15.34,
+      "learning_rate": 9.05263157894737e-06,
+      "loss": 4.9633,
+      "step": 157000
+    },
+    {
+      "epoch": 15.44,
+      "learning_rate": 8.842105263157895e-06,
+      "loss": 4.9613,
+      "step": 158000
+    },
+    {
+      "epoch": 15.53,
+      "learning_rate": 8.631578947368422e-06,
+      "loss": 4.9629,
+      "step": 159000
+    },
+    {
+      "epoch": 15.63,
+      "learning_rate": 8.421052631578948e-06,
+      "loss": 4.9547,
+      "step": 160000
+    },
+    {
+      "epoch": 15.73,
+      "learning_rate": 8.210526315789475e-06,
+      "loss": 4.9541,
+      "step": 161000
+    },
+    {
+      "epoch": 15.83,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 4.9532,
+      "step": 162000
+    },
+    {
+      "epoch": 15.93,
+      "learning_rate": 7.789473684210526e-06,
+      "loss": 4.9461,
+      "step": 163000
+    },
+    {
+      "epoch": 16.02,
+      "learning_rate": 7.578947368421054e-06,
+      "loss": 4.9518,
+      "step": 164000
+    },
+    {
+      "epoch": 16.12,
+      "learning_rate": 7.368421052631579e-06,
+      "loss": 4.9543,
+      "step": 165000
+    },
+    {
+      "epoch": 16.22,
+      "learning_rate": 7.157894736842106e-06,
+      "loss": 4.9526,
+      "step": 166000
+    },
+    {
+      "epoch": 16.32,
+      "learning_rate": 6.947368421052632e-06,
+      "loss": 4.9447,
+      "step": 167000
+    },
+    {
+      "epoch": 16.41,
+      "learning_rate": 6.736842105263158e-06,
+      "loss": 4.9545,
+      "step": 168000
+    },
+    {
+      "epoch": 16.51,
+      "learning_rate": 6.526315789473685e-06,
+      "loss": 4.9448,
+      "step": 169000
+    },
+    {
+      "epoch": 16.61,
+      "learning_rate": 6.31578947368421e-06,
+      "loss": 4.9521,
+      "step": 170000
+    },
+    {
+      "epoch": 16.71,
+      "learning_rate": 6.105263157894738e-06,
+      "loss": 4.9503,
+      "step": 171000
+    },
+    {
+      "epoch": 16.81,
+      "learning_rate": 5.8947368421052634e-06,
+      "loss": 4.9537,
+      "step": 172000
+    },
+    {
+      "epoch": 16.9,
+      "learning_rate": 5.68421052631579e-06,
+      "loss": 4.9446,
+      "step": 173000
+    },
+    {
+      "epoch": 17.0,
+      "learning_rate": 5.4736842105263165e-06,
+      "loss": 4.9397,
+      "step": 174000
+    },
+    {
+      "epoch": 17.1,
+      "learning_rate": 5.263157894736842e-06,
+      "loss": 4.9541,
+      "step": 175000
+    },
+    {
+      "epoch": 17.2,
+      "learning_rate": 5.052631578947369e-06,
+      "loss": 4.9449,
+      "step": 176000
+    },
+    {
+      "epoch": 17.29,
+      "learning_rate": 4.842105263157895e-06,
+      "loss": 4.9447,
+      "step": 177000
+    },
+    {
+      "epoch": 17.39,
+      "learning_rate": 4.631578947368421e-06,
+      "loss": 4.9344,
+      "step": 178000
+    },
+    {
+      "epoch": 17.49,
+      "learning_rate": 4.4210526315789476e-06,
+      "loss": 4.933,
+      "step": 179000
+    },
+    {
+      "epoch": 17.59,
+      "learning_rate": 4.210526315789474e-06,
+      "loss": 4.9397,
+      "step": 180000
+    },
+    {
+      "epoch": 17.68,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 4.9431,
+      "step": 181000
+    },
+    {
+      "epoch": 17.78,
+      "learning_rate": 3.789473684210527e-06,
+      "loss": 4.9378,
+      "step": 182000
+    },
+    {
+      "epoch": 17.88,
+      "learning_rate": 3.578947368421053e-06,
+      "loss": 4.9399,
+      "step": 183000
+    },
+    {
+      "epoch": 17.98,
+      "learning_rate": 3.368421052631579e-06,
+      "loss": 4.9481,
+      "step": 184000
+    },
+    {
+      "epoch": 18.08,
+      "learning_rate": 3.157894736842105e-06,
+      "loss": 4.9399,
+      "step": 185000
+    },
+    {
+      "epoch": 18.17,
+      "learning_rate": 2.9473684210526317e-06,
+      "loss": 4.9365,
+      "step": 186000
+    },
+    {
+      "epoch": 18.27,
+      "learning_rate": 2.7368421052631583e-06,
+      "loss": 4.9307,
+      "step": 187000
+    },
+    {
+      "epoch": 18.37,
+      "learning_rate": 2.5263157894736844e-06,
+      "loss": 4.9391,
+      "step": 188000
+    },
+    {
+      "epoch": 18.47,
+      "learning_rate": 2.3157894736842105e-06,
+      "loss": 4.9331,
+      "step": 189000
+    },
+    {
+      "epoch": 18.56,
+      "learning_rate": 2.105263157894737e-06,
+      "loss": 4.948,
+      "step": 190000
+    },
+    {
+      "epoch": 18.66,
+      "learning_rate": 1.8947368421052634e-06,
+      "loss": 4.9407,
+      "step": 191000
+    },
+    {
+      "epoch": 18.76,
+      "learning_rate": 1.6842105263157895e-06,
+      "loss": 4.944,
+      "step": 192000
+    },
+    {
+      "epoch": 18.86,
+      "learning_rate": 1.4736842105263159e-06,
+      "loss": 4.9286,
+      "step": 193000
+    },
+    {
+      "epoch": 18.95,
+      "learning_rate": 1.2631578947368422e-06,
+      "loss": 4.9338,
+      "step": 194000
+    }
+  ],
+  "logging_steps": 1000,
+  "max_steps": 200000,
+  "num_train_epochs": 20,
+  "save_steps": 1000,
+  "total_flos": 2.258245829660849e+19,
+  "trial_name": null,
+  "trial_params": null
+}

len2_5120/checkpoint-194000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9ff5d6388a59708cb6305a7fab984d438e385b1835680f9629b90c23c5cf704
+size 5137

len2_5120/checkpoint-195000/config.json

@@ -0,0 +1,25 @@
+{
+  "architectures": [
+    "BertForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_length": 512,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.35.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 5120
+}

len2_5120/checkpoint-195000/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "max_length": 512,
+  "pad_token_id": 0,
+  "transformers_version": "4.35.2"
+}

len2_5120/checkpoint-195000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f08acae5c42aec4d08dd4c07ecf0003d0906dac4b6898a7a28eaa17d433b6b37
+size 359944256

len2_5120/checkpoint-195000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:557e5f514efd168ec0a2e047f5f68ef4d0d9419d2589e23428e517eb9b2fcc41
+size 720010699

len2_5120/checkpoint-195000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c417c0f604cac9e266875d12cb57ca16657d4c563a05f5a5e3e6ef9004ab158
+size 14645

len2_5120/checkpoint-195000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac3001d5dd025a8da6bc02910e07eb74e4b31d248926c79f2143788adccb7d21
+size 1465

len2_5120/checkpoint-195000/trainer_state.json

@@ -0,0 +1,1189 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 19.05227161700049,
+  "eval_steps": 500,
+  "global_step": 195000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.78,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 7.5832,
+      "step": 1000
+    },
+    {
+      "epoch": 1.56,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 6.9724,
+      "step": 2000
+    },
+    {
+      "epoch": 2.34,
+      "learning_rate": 1.2e-05,
+      "loss": 6.7695,
+      "step": 3000
+    },
+    {
+      "epoch": 3.12,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 6.7002,
+      "step": 4000
+    },
+    {
+      "epoch": 3.91,
+      "learning_rate": 2e-05,
+      "loss": 6.666,
+      "step": 5000
+    },
+    {
+      "epoch": 4.69,
+      "learning_rate": 2.4e-05,
+      "loss": 6.6503,
+      "step": 6000
+    },
+    {
+      "epoch": 5.47,
+      "learning_rate": 2.8e-05,
+      "loss": 6.6407,
+      "step": 7000
+    },
+    {
+      "epoch": 6.25,
+      "learning_rate": 3.2000000000000005e-05,
+      "loss": 6.6277,
+      "step": 8000
+    },
+    {
+      "epoch": 7.03,
+      "learning_rate": 3.6e-05,
+      "loss": 6.6209,
+      "step": 9000
+    },
+    {
+      "epoch": 7.81,
+      "learning_rate": 4e-05,
+      "loss": 6.6135,
+      "step": 10000
+    },
+    {
+      "epoch": 8.59,
+      "learning_rate": 3.955555555555556e-05,
+      "loss": 6.6084,
+      "step": 11000
+    },
+    {
+      "epoch": 9.38,
+      "learning_rate": 3.9111111111111115e-05,
+      "loss": 6.6041,
+      "step": 12000
+    },
+    {
+      "epoch": 10.16,
+      "learning_rate": 3.866666666666667e-05,
+      "loss": 6.6013,
+      "step": 13000
+    },
+    {
+      "epoch": 10.94,
+      "learning_rate": 3.8222222222222226e-05,
+      "loss": 6.5994,
+      "step": 14000
+    },
+    {
+      "epoch": 11.72,
+      "learning_rate": 3.777777777777778e-05,
+      "loss": 6.594,
+      "step": 15000
+    },
+    {
+      "epoch": 12.5,
+      "learning_rate": 3.733333333333334e-05,
+      "loss": 6.5925,
+      "step": 16000
+    },
+    {
+      "epoch": 13.28,
+      "learning_rate": 3.6888888888888896e-05,
+      "loss": 6.5893,
+      "step": 17000
+    },
+    {
+      "epoch": 14.06,
+      "learning_rate": 3.644444444444445e-05,
+      "loss": 6.5875,
+      "step": 18000
+    },
+    {
+      "epoch": 14.84,
+      "learning_rate": 3.6e-05,
+      "loss": 6.5232,
+      "step": 19000
+    },
+    {
+      "epoch": 15.62,
+      "learning_rate": 3.555555555555555e-05,
+      "loss": 6.4358,
+      "step": 20000
+    },
+    {
+      "epoch": 16.41,
+      "learning_rate": 3.511111111111111e-05,
+      "loss": 6.3712,
+      "step": 21000
+    },
+    {
+      "epoch": 17.19,
+      "learning_rate": 3.466666666666667e-05,
+      "loss": 6.311,
+      "step": 22000
+    },
+    {
+      "epoch": 17.97,
+      "learning_rate": 3.4222222222222224e-05,
+      "loss": 6.2493,
+      "step": 23000
+    },
+    {
+      "epoch": 18.75,
+      "learning_rate": 3.377777777777778e-05,
+      "loss": 6.1597,
+      "step": 24000
+    },
+    {
+      "epoch": 19.53,
+      "learning_rate": 3.3333333333333335e-05,
+      "loss": 6.0276,
+      "step": 25000
+    },
+    {
+      "epoch": 20.31,
+      "learning_rate": 3.288888888888889e-05,
+      "loss": 5.8317,
+      "step": 26000
+    },
+    {
+      "epoch": 21.09,
+      "learning_rate": 3.2444444444444446e-05,
+      "loss": 5.6782,
+      "step": 27000
+    },
+    {
+      "epoch": 21.88,
+      "learning_rate": 3.2000000000000005e-05,
+      "loss": 5.5975,
+      "step": 28000
+    },
+    {
+      "epoch": 22.66,
+      "learning_rate": 3.155555555555556e-05,
+      "loss": 5.5369,
+      "step": 29000
+    },
+    {
+      "epoch": 23.44,
+      "learning_rate": 3.111111111111112e-05,
+      "loss": 5.4921,
+      "step": 30000
+    },
+    {
+      "epoch": 24.22,
+      "learning_rate": 3.066666666666667e-05,
+      "loss": 5.4554,
+      "step": 31000
+    },
+    {
+      "epoch": 25.0,
+      "learning_rate": 3.0222222222222225e-05,
+      "loss": 5.4237,
+      "step": 32000
+    },
+    {
+      "epoch": 25.78,
+      "learning_rate": 2.977777777777778e-05,
+      "loss": 5.3938,
+      "step": 33000
+    },
+    {
+      "epoch": 26.56,
+      "learning_rate": 2.9333333333333333e-05,
+      "loss": 5.3701,
+      "step": 34000
+    },
+    {
+      "epoch": 27.34,
+      "learning_rate": 2.888888888888889e-05,
+      "loss": 5.3452,
+      "step": 35000
+    },
+    {
+      "epoch": 28.12,
+      "learning_rate": 2.8444444444444447e-05,
+      "loss": 5.327,
+      "step": 36000
+    },
+    {
+      "epoch": 28.91,
+      "learning_rate": 2.8e-05,
+      "loss": 5.3063,
+      "step": 37000
+    },
+    {
+      "epoch": 29.69,
+      "learning_rate": 2.755555555555556e-05,
+      "loss": 5.2905,
+      "step": 38000
+    },
+    {
+      "epoch": 30.47,
+      "learning_rate": 2.7111111111111114e-05,
+      "loss": 5.2747,
+      "step": 39000
+    },
+    {
+      "epoch": 31.25,
+      "learning_rate": 2.6666666666666667e-05,
+      "loss": 5.2596,
+      "step": 40000
+    },
+    {
+      "epoch": 32.03,
+      "learning_rate": 2.6222222222222226e-05,
+      "loss": 5.2483,
+      "step": 41000
+    },
+    {
+      "epoch": 32.81,
+      "learning_rate": 2.577777777777778e-05,
+      "loss": 5.2336,
+      "step": 42000
+    },
+    {
+      "epoch": 33.59,
+      "learning_rate": 2.5333333333333334e-05,
+      "loss": 5.2214,
+      "step": 43000
+    },
+    {
+      "epoch": 34.38,
+      "learning_rate": 2.4888888888888893e-05,
+      "loss": 5.2103,
+      "step": 44000
+    },
+    {
+      "epoch": 35.16,
+      "learning_rate": 2.444444444444445e-05,
+      "loss": 5.2015,
+      "step": 45000
+    },
+    {
+      "epoch": 35.94,
+      "learning_rate": 2.4e-05,
+      "loss": 5.1907,
+      "step": 46000
+    },
+    {
+      "epoch": 36.72,
+      "learning_rate": 2.355555555555556e-05,
+      "loss": 5.1801,
+      "step": 47000
+    },
+    {
+      "epoch": 37.5,
+      "learning_rate": 2.3111111111111112e-05,
+      "loss": 5.1713,
+      "step": 48000
+    },
+    {
+      "epoch": 38.28,
+      "learning_rate": 2.2666666666666668e-05,
+      "loss": 5.1645,
+      "step": 49000
+    },
+    {
+      "epoch": 39.06,
+      "learning_rate": 2.2222222222222227e-05,
+      "loss": 5.1548,
+      "step": 50000
+    },
+    {
+      "epoch": 39.84,
+      "learning_rate": 2.177777777777778e-05,
+      "loss": 5.1462,
+      "step": 51000
+    },
+    {
+      "epoch": 40.62,
+      "learning_rate": 2.1333333333333335e-05,
+      "loss": 5.1417,
+      "step": 52000
+    },
+    {
+      "epoch": 41.41,
+      "learning_rate": 2.088888888888889e-05,
+      "loss": 5.1305,
+      "step": 53000
+    },
+    {
+      "epoch": 42.19,
+      "learning_rate": 2.0444444444444446e-05,
+      "loss": 5.1287,
+      "step": 54000
+    },
+    {
+      "epoch": 42.97,
+      "learning_rate": 2e-05,
+      "loss": 5.1202,
+      "step": 55000
+    },
+    {
+      "epoch": 43.75,
+      "learning_rate": 1.9555555555555557e-05,
+      "loss": 5.1139,
+      "step": 56000
+    },
+    {
+      "epoch": 44.53,
+      "learning_rate": 1.9111111111111113e-05,
+      "loss": 5.1081,
+      "step": 57000
+    },
+    {
+      "epoch": 45.31,
+      "learning_rate": 1.866666666666667e-05,
+      "loss": 5.1032,
+      "step": 58000
+    },
+    {
+      "epoch": 46.09,
+      "learning_rate": 1.8222222222222224e-05,
+      "loss": 5.0968,
+      "step": 59000
+    },
+    {
+      "epoch": 46.88,
+      "learning_rate": 1.7777777777777777e-05,
+      "loss": 5.0914,
+      "step": 60000
+    },
+    {
+      "epoch": 47.66,
+      "learning_rate": 1.7333333333333336e-05,
+      "loss": 5.085,
+      "step": 61000
+    },
+    {
+      "epoch": 48.44,
+      "learning_rate": 1.688888888888889e-05,
+      "loss": 5.0806,
+      "step": 62000
+    },
+    {
+      "epoch": 49.22,
+      "learning_rate": 1.6444444444444444e-05,
+      "loss": 5.0769,
+      "step": 63000
+    },
+    {
+      "epoch": 50.0,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 5.0728,
+      "step": 64000
+    },
+    {
+      "epoch": 50.78,
+      "learning_rate": 1.555555555555556e-05,
+      "loss": 5.0692,
+      "step": 65000
+    },
+    {
+      "epoch": 51.56,
+      "learning_rate": 1.5111111111111112e-05,
+      "loss": 5.065,
+      "step": 66000
+    },
+    {
+      "epoch": 52.34,
+      "learning_rate": 1.4666666666666666e-05,
+      "loss": 5.0594,
+      "step": 67000
+    },
+    {
+      "epoch": 53.12,
+      "learning_rate": 1.4222222222222224e-05,
+      "loss": 5.0549,
+      "step": 68000
+    },
+    {
+      "epoch": 53.91,
+      "learning_rate": 1.377777777777778e-05,
+      "loss": 5.0538,
+      "step": 69000
+    },
+    {
+      "epoch": 54.69,
+      "learning_rate": 1.3333333333333333e-05,
+      "loss": 5.0482,
+      "step": 70000
+    },
+    {
+      "epoch": 55.47,
+      "learning_rate": 1.288888888888889e-05,
+      "loss": 5.0475,
+      "step": 71000
+    },
+    {
+      "epoch": 56.25,
+      "learning_rate": 1.2444444444444446e-05,
+      "loss": 5.0399,
+      "step": 72000
+    },
+    {
+      "epoch": 57.03,
+      "learning_rate": 1.2e-05,
+      "loss": 5.0406,
+      "step": 73000
+    },
+    {
+      "epoch": 57.81,
+      "learning_rate": 1.1555555555555556e-05,
+      "loss": 5.037,
+      "step": 74000
+    },
+    {
+      "epoch": 58.59,
+      "learning_rate": 1.1111111111111113e-05,
+      "loss": 5.0326,
+      "step": 75000
+    },
+    {
+      "epoch": 59.38,
+      "learning_rate": 1.0666666666666667e-05,
+      "loss": 5.031,
+      "step": 76000
+    },
+    {
+      "epoch": 60.16,
+      "learning_rate": 1.0222222222222223e-05,
+      "loss": 5.0291,
+      "step": 77000
+    },
+    {
+      "epoch": 60.94,
+      "learning_rate": 9.777777777777779e-06,
+      "loss": 5.0266,
+      "step": 78000
+    },
+    {
+      "epoch": 61.72,
+      "learning_rate": 9.333333333333334e-06,
+      "loss": 5.0235,
+      "step": 79000
+    },
+    {
+      "epoch": 62.5,
+      "learning_rate": 8.888888888888888e-06,
+      "loss": 5.0211,
+      "step": 80000
+    },
+    {
+      "epoch": 63.28,
+      "learning_rate": 8.444444444444446e-06,
+      "loss": 5.0214,
+      "step": 81000
+    },
+    {
+      "epoch": 64.06,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 5.0176,
+      "step": 82000
+    },
+    {
+      "epoch": 64.84,
+      "learning_rate": 7.555555555555556e-06,
+      "loss": 5.0157,
+      "step": 83000
+    },
+    {
+      "epoch": 65.62,
+      "learning_rate": 7.111111111111112e-06,
+      "loss": 5.014,
+      "step": 84000
+    },
+    {
+      "epoch": 66.41,
+      "learning_rate": 6.666666666666667e-06,
+      "loss": 5.0115,
+      "step": 85000
+    },
+    {
+      "epoch": 67.19,
+      "learning_rate": 6.222222222222223e-06,
+      "loss": 5.0087,
+      "step": 86000
+    },
+    {
+      "epoch": 67.97,
+      "learning_rate": 5.777777777777778e-06,
+      "loss": 5.0105,
+      "step": 87000
+    },
+    {
+      "epoch": 68.75,
+      "learning_rate": 5.333333333333334e-06,
+      "loss": 5.0062,
+      "step": 88000
+    },
+    {
+      "epoch": 69.53,
+      "learning_rate": 4.888888888888889e-06,
+      "loss": 5.0028,
+      "step": 89000
+    },
+    {
+      "epoch": 70.31,
+      "learning_rate": 4.444444444444444e-06,
+      "loss": 5.0066,
+      "step": 90000
+    },
+    {
+      "epoch": 71.09,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 5.0032,
+      "step": 91000
+    },
+    {
+      "epoch": 71.88,
+      "learning_rate": 3.555555555555556e-06,
+      "loss": 5.0022,
+      "step": 92000
+    },
+    {
+      "epoch": 72.66,
+      "learning_rate": 3.1111111111111116e-06,
+      "loss": 5.0024,
+      "step": 93000
+    },
+    {
+      "epoch": 73.44,
+      "learning_rate": 2.666666666666667e-06,
+      "loss": 5.0007,
+      "step": 94000
+    },
+    {
+      "epoch": 74.22,
+      "learning_rate": 2.222222222222222e-06,
+      "loss": 5.0008,
+      "step": 95000
+    },
+    {
+      "epoch": 75.0,
+      "learning_rate": 1.777777777777778e-06,
+      "loss": 4.9978,
+      "step": 96000
+    },
+    {
+      "epoch": 75.78,
+      "learning_rate": 1.3333333333333334e-06,
+      "loss": 4.9966,
+      "step": 97000
+    },
+    {
+      "epoch": 76.56,
+      "learning_rate": 8.88888888888889e-07,
+      "loss": 4.9994,
+      "step": 98000
+    },
+    {
+      "epoch": 77.34,
+      "learning_rate": 4.444444444444445e-07,
+      "loss": 4.9966,
+      "step": 99000
+    },
+    {
+      "epoch": 78.12,
+      "learning_rate": 0.0,
+      "loss": 4.9976,
+      "step": 100000
+    },
+    {
+      "epoch": 9.87,
+      "learning_rate": 2.0842105263157895e-05,
+      "loss": 5.0155,
+      "step": 101000
+    },
+    {
+      "epoch": 9.97,
+      "learning_rate": 2.0631578947368425e-05,
+      "loss": 5.024,
+      "step": 102000
+    },
+    {
+      "epoch": 10.06,
+      "learning_rate": 2.0421052631578952e-05,
+      "loss": 5.0241,
+      "step": 103000
+    },
+    {
+      "epoch": 10.16,
+      "learning_rate": 2.0210526315789475e-05,
+      "loss": 5.0218,
+      "step": 104000
+    },
+    {
+      "epoch": 10.26,
+      "learning_rate": 2e-05,
+      "loss": 5.0231,
+      "step": 105000
+    },
+    {
+      "epoch": 10.36,
+      "learning_rate": 1.9789473684210528e-05,
+      "loss": 5.0231,
+      "step": 106000
+    },
+    {
+      "epoch": 10.45,
+      "learning_rate": 1.9578947368421055e-05,
+      "loss": 5.0198,
+      "step": 107000
+    },
+    {
+      "epoch": 10.55,
+      "learning_rate": 1.936842105263158e-05,
+      "loss": 5.0158,
+      "step": 108000
+    },
+    {
+      "epoch": 10.65,
+      "learning_rate": 1.9157894736842108e-05,
+      "loss": 5.0122,
+      "step": 109000
+    },
+    {
+      "epoch": 10.75,
+      "learning_rate": 1.894736842105263e-05,
+      "loss": 5.0165,
+      "step": 110000
+    },
+    {
+      "epoch": 10.85,
+      "learning_rate": 1.873684210526316e-05,
+      "loss": 5.0183,
+      "step": 111000
+    },
+    {
+      "epoch": 10.94,
+      "learning_rate": 1.8526315789473684e-05,
+      "loss": 5.01,
+      "step": 112000
+    },
+    {
+      "epoch": 11.04,
+      "learning_rate": 1.831578947368421e-05,
+      "loss": 5.007,
+      "step": 113000
+    },
+    {
+      "epoch": 11.14,
+      "learning_rate": 1.810526315789474e-05,
+      "loss": 5.0059,
+      "step": 114000
+    },
+    {
+      "epoch": 11.24,
+      "learning_rate": 1.7894736842105264e-05,
+      "loss": 5.0089,
+      "step": 115000
+    },
+    {
+      "epoch": 11.33,
+      "learning_rate": 1.768421052631579e-05,
+      "loss": 5.0081,
+      "step": 116000
+    },
+    {
+      "epoch": 11.43,
+      "learning_rate": 1.7473684210526317e-05,
+      "loss": 5.002,
+      "step": 117000
+    },
+    {
+      "epoch": 11.53,
+      "learning_rate": 1.7263157894736843e-05,
+      "loss": 5.0052,
+      "step": 118000
+    },
+    {
+      "epoch": 11.63,
+      "learning_rate": 1.705263157894737e-05,
+      "loss": 5.0079,
+      "step": 119000
+    },
+    {
+      "epoch": 11.72,
+      "learning_rate": 1.6842105263157896e-05,
+      "loss": 4.9985,
+      "step": 120000
+    },
+    {
+      "epoch": 11.82,
+      "learning_rate": 1.6631578947368423e-05,
+      "loss": 5.006,
+      "step": 121000
+    },
+    {
+      "epoch": 11.92,
+      "learning_rate": 1.642105263157895e-05,
+      "loss": 4.9956,
+      "step": 122000
+    },
+    {
+      "epoch": 12.02,
+      "learning_rate": 1.6210526315789473e-05,
+      "loss": 4.9973,
+      "step": 123000
+    },
+    {
+      "epoch": 12.12,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 4.9976,
+      "step": 124000
+    },
+    {
+      "epoch": 12.21,
+      "learning_rate": 1.578947368421053e-05,
+      "loss": 4.9882,
+      "step": 125000
+    },
+    {
+      "epoch": 12.31,
+      "learning_rate": 1.5578947368421052e-05,
+      "loss": 4.9908,
+      "step": 126000
+    },
+    {
+      "epoch": 12.41,
+      "learning_rate": 1.536842105263158e-05,
+      "loss": 5.0038,
+      "step": 127000
+    },
+    {
+      "epoch": 12.51,
+      "learning_rate": 1.5157894736842107e-05,
+      "loss": 4.9923,
+      "step": 128000
+    },
+    {
+      "epoch": 12.6,
+      "learning_rate": 1.4947368421052632e-05,
+      "loss": 4.9847,
+      "step": 129000
+    },
+    {
+      "epoch": 12.7,
+      "learning_rate": 1.4736842105263159e-05,
+      "loss": 4.9864,
+      "step": 130000
+    },
+    {
+      "epoch": 12.8,
+      "learning_rate": 1.4526315789473687e-05,
+      "loss": 4.9903,
+      "step": 131000
+    },
+    {
+      "epoch": 12.9,
+      "learning_rate": 1.4315789473684212e-05,
+      "loss": 4.9812,
+      "step": 132000
+    },
+    {
+      "epoch": 12.99,
+      "learning_rate": 1.4105263157894738e-05,
+      "loss": 4.9923,
+      "step": 133000
+    },
+    {
+      "epoch": 13.09,
+      "learning_rate": 1.3894736842105265e-05,
+      "loss": 4.9838,
+      "step": 134000
+    },
+    {
+      "epoch": 13.19,
+      "learning_rate": 1.3684210526315791e-05,
+      "loss": 4.977,
+      "step": 135000
+    },
+    {
+      "epoch": 13.29,
+      "learning_rate": 1.3473684210526316e-05,
+      "loss": 4.9743,
+      "step": 136000
+    },
+    {
+      "epoch": 13.39,
+      "learning_rate": 1.3263157894736843e-05,
+      "loss": 4.9797,
+      "step": 137000
+    },
+    {
+      "epoch": 13.48,
+      "learning_rate": 1.305263157894737e-05,
+      "loss": 4.9915,
+      "step": 138000
+    },
+    {
+      "epoch": 13.58,
+      "learning_rate": 1.2842105263157896e-05,
+      "loss": 4.9762,
+      "step": 139000
+    },
+    {
+      "epoch": 13.68,
+      "learning_rate": 1.263157894736842e-05,
+      "loss": 4.9771,
+      "step": 140000
+    },
+    {
+      "epoch": 13.78,
+      "learning_rate": 1.2421052631578949e-05,
+      "loss": 4.9763,
+      "step": 141000
+    },
+    {
+      "epoch": 13.87,
+      "learning_rate": 1.2210526315789475e-05,
+      "loss": 4.9706,
+      "step": 142000
+    },
+    {
+      "epoch": 13.97,
+      "learning_rate": 1.2e-05,
+      "loss": 4.9731,
+      "step": 143000
+    },
+    {
+      "epoch": 14.07,
+      "learning_rate": 1.1789473684210527e-05,
+      "loss": 4.9736,
+      "step": 144000
+    },
+    {
+      "epoch": 14.17,
+      "learning_rate": 1.1578947368421053e-05,
+      "loss": 4.9704,
+      "step": 145000
+    },
+    {
+      "epoch": 14.26,
+      "learning_rate": 1.136842105263158e-05,
+      "loss": 4.9762,
+      "step": 146000
+    },
+    {
+      "epoch": 14.36,
+      "learning_rate": 1.1157894736842105e-05,
+      "loss": 4.9709,
+      "step": 147000
+    },
+    {
+      "epoch": 14.46,
+      "learning_rate": 1.0947368421052633e-05,
+      "loss": 4.957,
+      "step": 148000
+    },
+    {
+      "epoch": 14.56,
+      "learning_rate": 1.073684210526316e-05,
+      "loss": 4.9666,
+      "step": 149000
+    },
+    {
+      "epoch": 14.66,
+      "learning_rate": 1.0526315789473684e-05,
+      "loss": 4.9623,
+      "step": 150000
+    },
+    {
+      "epoch": 14.75,
+      "learning_rate": 1.0315789473684213e-05,
+      "loss": 4.9613,
+      "step": 151000
+    },
+    {
+      "epoch": 14.85,
+      "learning_rate": 1.0105263157894738e-05,
+      "loss": 4.9642,
+      "step": 152000
+    },
+    {
+      "epoch": 14.95,
+      "learning_rate": 9.894736842105264e-06,
+      "loss": 4.9605,
+      "step": 153000
+    },
+    {
+      "epoch": 15.05,
+      "learning_rate": 9.68421052631579e-06,
+      "loss": 4.9688,
+      "step": 154000
+    },
+    {
+      "epoch": 15.14,
+      "learning_rate": 9.473684210526315e-06,
+      "loss": 4.9635,
+      "step": 155000
+    },
+    {
+      "epoch": 15.24,
+      "learning_rate": 9.263157894736842e-06,
+      "loss": 4.9587,
+      "step": 156000
+    },
+    {
+      "epoch": 15.34,
+      "learning_rate": 9.05263157894737e-06,
+      "loss": 4.9633,
+      "step": 157000
+    },
+    {
+      "epoch": 15.44,
+      "learning_rate": 8.842105263157895e-06,
+      "loss": 4.9613,
+      "step": 158000
+    },
+    {
+      "epoch": 15.53,
+      "learning_rate": 8.631578947368422e-06,
+      "loss": 4.9629,
+      "step": 159000
+    },
+    {
+      "epoch": 15.63,
+      "learning_rate": 8.421052631578948e-06,
+      "loss": 4.9547,
+      "step": 160000
+    },
+    {
+      "epoch": 15.73,
+      "learning_rate": 8.210526315789475e-06,
+      "loss": 4.9541,
+      "step": 161000
+    },
+    {
+      "epoch": 15.83,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 4.9532,
+      "step": 162000
+    },
+    {
+      "epoch": 15.93,
+      "learning_rate": 7.789473684210526e-06,
+      "loss": 4.9461,
+      "step": 163000
+    },
+    {
+      "epoch": 16.02,
+      "learning_rate": 7.578947368421054e-06,
+      "loss": 4.9518,
+      "step": 164000
+    },
+    {
+      "epoch": 16.12,
+      "learning_rate": 7.368421052631579e-06,
+      "loss": 4.9543,
+      "step": 165000
+    },
+    {
+      "epoch": 16.22,
+      "learning_rate": 7.157894736842106e-06,
+      "loss": 4.9526,
+      "step": 166000
+    },
+    {
+      "epoch": 16.32,
+      "learning_rate": 6.947368421052632e-06,
+      "loss": 4.9447,
+      "step": 167000
+    },
+    {
+      "epoch": 16.41,
+      "learning_rate": 6.736842105263158e-06,
+      "loss": 4.9545,
+      "step": 168000
+    },
+    {
+      "epoch": 16.51,
+      "learning_rate": 6.526315789473685e-06,
+      "loss": 4.9448,
+      "step": 169000
+    },
+    {
+      "epoch": 16.61,
+      "learning_rate": 6.31578947368421e-06,
+      "loss": 4.9521,
+      "step": 170000
+    },
+    {
+      "epoch": 16.71,
+      "learning_rate": 6.105263157894738e-06,
+      "loss": 4.9503,
+      "step": 171000
+    },
+    {
+      "epoch": 16.81,
+      "learning_rate": 5.8947368421052634e-06,
+      "loss": 4.9537,
+      "step": 172000
+    },
+    {
+      "epoch": 16.9,
+      "learning_rate": 5.68421052631579e-06,
+      "loss": 4.9446,
+      "step": 173000
+    },
+    {
+      "epoch": 17.0,
+      "learning_rate": 5.4736842105263165e-06,
+      "loss": 4.9397,
+      "step": 174000
+    },
+    {
+      "epoch": 17.1,
+      "learning_rate": 5.263157894736842e-06,
+      "loss": 4.9541,
+      "step": 175000
+    },
+    {
+      "epoch": 17.2,
+      "learning_rate": 5.052631578947369e-06,
+      "loss": 4.9449,
+      "step": 176000
+    },
+    {
+      "epoch": 17.29,
+      "learning_rate": 4.842105263157895e-06,
+      "loss": 4.9447,
+      "step": 177000
+    },
+    {
+      "epoch": 17.39,
+      "learning_rate": 4.631578947368421e-06,
+      "loss": 4.9344,
+      "step": 178000
+    },
+    {
+      "epoch": 17.49,
+      "learning_rate": 4.4210526315789476e-06,
+      "loss": 4.933,
+      "step": 179000
+    },
+    {
+      "epoch": 17.59,
+      "learning_rate": 4.210526315789474e-06,
+      "loss": 4.9397,
+      "step": 180000
+    },
+    {
+      "epoch": 17.68,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 4.9431,
+      "step": 181000
+    },
+    {
+      "epoch": 17.78,
+      "learning_rate": 3.789473684210527e-06,
+      "loss": 4.9378,
+      "step": 182000
+    },
+    {
+      "epoch": 17.88,
+      "learning_rate": 3.578947368421053e-06,
+      "loss": 4.9399,
+      "step": 183000
+    },
+    {
+      "epoch": 17.98,
+      "learning_rate": 3.368421052631579e-06,
+      "loss": 4.9481,
+      "step": 184000
+    },
+    {
+      "epoch": 18.08,
+      "learning_rate": 3.157894736842105e-06,
+      "loss": 4.9399,
+      "step": 185000
+    },
+    {
+      "epoch": 18.17,
+      "learning_rate": 2.9473684210526317e-06,
+      "loss": 4.9365,
+      "step": 186000
+    },
+    {
+      "epoch": 18.27,
+      "learning_rate": 2.7368421052631583e-06,
+      "loss": 4.9307,
+      "step": 187000
+    },
+    {
+      "epoch": 18.37,
+      "learning_rate": 2.5263157894736844e-06,
+      "loss": 4.9391,
+      "step": 188000
+    },
+    {
+      "epoch": 18.47,
+      "learning_rate": 2.3157894736842105e-06,
+      "loss": 4.9331,
+      "step": 189000
+    },
+    {
+      "epoch": 18.56,
+      "learning_rate": 2.105263157894737e-06,
+      "loss": 4.948,
+      "step": 190000
+    },
+    {
+      "epoch": 18.66,
+      "learning_rate": 1.8947368421052634e-06,
+      "loss": 4.9407,
+      "step": 191000
+    },
+    {
+      "epoch": 18.76,
+      "learning_rate": 1.6842105263157895e-06,
+      "loss": 4.944,
+      "step": 192000
+    },
+    {
+      "epoch": 18.86,
+      "learning_rate": 1.4736842105263159e-06,
+      "loss": 4.9286,
+      "step": 193000
+    },
+    {
+      "epoch": 18.95,
+      "learning_rate": 1.2631578947368422e-06,
+      "loss": 4.9338,
+      "step": 194000
+    },
+    {
+      "epoch": 19.05,
+      "learning_rate": 1.0526315789473685e-06,
+      "loss": 4.9357,
+      "step": 195000
+    }
+  ],
+  "logging_steps": 1000,
+  "max_steps": 200000,
+  "num_train_epochs": 20,
+  "save_steps": 1000,
+  "total_flos": 2.2607703469056328e+19,
+  "trial_name": null,
+  "trial_params": null
+}

len2_5120/checkpoint-195000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9ff5d6388a59708cb6305a7fab984d438e385b1835680f9629b90c23c5cf704
+size 5137

len2_5120/checkpoint-196000/config.json

@@ -0,0 +1,25 @@
+{
+  "architectures": [
+    "BertForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_length": 512,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.35.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 5120
+}

len2_5120/checkpoint-196000/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "max_length": 512,
+  "pad_token_id": 0,
+  "transformers_version": "4.35.2"
+}

len2_5120/checkpoint-196000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93b5c27f5deb7b1b609b2d5cc9ea6c0929507d7419b4a7b7c5c13656d66d9b0f
+size 359944256

len2_5120/checkpoint-196000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73372e579073fdfb073e755d42fa884d0e55d89671ba5e0fac500afa0ac9c72b
+size 720010699

len2_5120/checkpoint-196000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fafaef33931b73eb438d9dcbd75cbb6a779009b5570372ee03872e55b8f5690c
+size 14645