Hugging Face's logo

codegenstudio
/
codegen-350M-nosql2doc-lora

Text Generation
PEFT
Safetensors
lora
nosql2doc
Model card Files
xet
 Community
bhavanikambhampati commited on
Commit
aac6fc6
·
verified ·
1 Parent(s): bb3c78d

Publish nosql2doc adapter (v3)

Browse files
Files changed (17) hide show
  1. README.md +8 -10
  2. adapter_config.json +45 -43
  3. adapter_model.safetensors +2 -2
  4. checkpoint-1509/README.md +209 -0
  5. checkpoint-1509/adapter_config.json +45 -0
  6. checkpoint-1509/adapter_model.safetensors +3 -0
  7. checkpoint-1509/optimizer.pt +3 -0
  8. checkpoint-1509/rng_state.pth +3 -0
  9. checkpoint-1509/scheduler.pt +3 -0
  10. checkpoint-1509/tokenizer.json +0 -0
  11. checkpoint-1509/tokenizer_config.json +11 -0
  12. checkpoint-1509/trainer_state.json +1576 -0
  13. checkpoint-1509/training_args.bin +3 -0
  14. run_metadata.json +3127 -0
  15. tokenizer.json +0 -0
  16. tokenizer_config.json +11 -0
  17. training_args.bin +3 -0
README.md CHANGED
@@ -1,20 +1,18 @@
1
  ---
2
- library_name: peft
3
  base_model: Salesforce/codegen-350M-multi
4
- tags:
5
- - lora
6
- - peft
7
- - code
8
- - nosql2doc
9
- - codegen
10
  pipeline_tag: text-generation
 
 
 
 
11
  ---
12
 
13
  # codegenstudio/codegen-350M-nosql2doc-lora
14
 
15
- LoRA adapter for **nosql2doc** on [Salesforce/codegen-350M-multi](https://huggingface.co/Salesforce/codegen-350M-multi).
16
 
17
- - Checkpoint version: `v2`
18
  - Task: `nosql2doc`
19
 
20
  ## Usage
@@ -31,4 +29,4 @@ model = AutoModelForCausalLM.from_pretrained(base)
31
  model = PeftModel.from_pretrained(model, adapter)
32
  ```
33
 
34
- Or use the multi-adapter API in this project's `hf-deploy/` package.
 
1
  ---
 
2
  base_model: Salesforce/codegen-350M-multi
3
+ library_name: peft
 
 
 
 
 
4
  pipeline_tag: text-generation
5
+ tags:
6
+ - lora
7
+ - peft
8
+ - nosql2doc
9
  ---
10
 
11
  # codegenstudio/codegen-350M-nosql2doc-lora
12
 
13
+ LoRA adapter for nosql2doc on Salesforce/codegen-350M-multi.
14
 
15
+ - Checkpoint version: `v3`
16
  - Task: `nosql2doc`
17
 
18
  ## Usage
 
29
  model = PeftModel.from_pretrained(model, adapter)
30
  ```
31
 
32
+ Or use the multi-adapter API in this project's `fastapi-deploy/` package.
adapter_config.json CHANGED
@@ -1,43 +1,45 @@
1
- {
2
- "alora_invocation_tokens": null,
3
- "alpha_pattern": {},
4
- "arrow_config": null,
5
- "auto_mapping": null,
6
- "base_model_name_or_path": "Salesforce/codegen-350M-multi",
7
- "bias": "none",
8
- "corda_config": null,
9
- "ensure_weight_tying": false,
10
- "eva_config": null,
11
- "exclude_modules": null,
12
- "fan_in_fan_out": false,
13
- "inference_mode": true,
14
- "init_lora_weights": true,
15
- "layer_replication": null,
16
- "layers_pattern": null,
17
- "layers_to_transform": null,
18
- "loftq_config": {},
19
- "lora_alpha": 32,
20
- "lora_bias": false,
21
- "lora_dropout": 0.05,
22
- "lora_ga_config": null,
23
- "megatron_config": null,
24
- "megatron_core": "megatron.core",
25
- "modules_to_save": null,
26
- "peft_type": "LORA",
27
- "peft_version": "0.19.1",
28
- "qalora_group_size": 16,
29
- "r": 16,
30
- "rank_pattern": {},
31
- "revision": null,
32
- "target_modules": [
33
- "qkv_proj",
34
- "out_proj"
35
- ],
36
- "target_parameters": null,
37
- "task_type": "CAUSAL_LM",
38
- "trainable_token_indices": null,
39
- "use_bdlora": null,
40
- "use_dora": false,
41
- "use_qalora": false,
42
- "use_rslora": false
43
- }
 
 
 
1
+ {
2
+ "alora_invocation_tokens": null,
3
+ "alpha_pattern": {},
4
+ "arrow_config": null,
5
+ "auto_mapping": null,
6
+ "base_model_name_or_path": "/kaggle/working/models/base/Salesforce__codegen-350M-multi",
7
+ "bias": "none",
8
+ "corda_config": null,
9
+ "ensure_weight_tying": false,
10
+ "eva_config": null,
11
+ "exclude_modules": null,
12
+ "fan_in_fan_out": false,
13
+ "inference_mode": true,
14
+ "init_lora_weights": true,
15
+ "layer_replication": null,
16
+ "layers_pattern": null,
17
+ "layers_to_transform": null,
18
+ "loftq_config": {},
19
+ "lora_alpha": 64,
20
+ "lora_bias": false,
21
+ "lora_dropout": 0.05,
22
+ "lora_ga_config": null,
23
+ "megatron_config": null,
24
+ "megatron_core": "megatron.core",
25
+ "modules_to_save": null,
26
+ "peft_type": "LORA",
27
+ "peft_version": "0.19.1",
28
+ "qalora_group_size": 16,
29
+ "r": 32,
30
+ "rank_pattern": {},
31
+ "revision": null,
32
+ "target_modules": [
33
+ "qkv_proj",
34
+ "out_proj",
35
+ "fc_in",
36
+ "fc_out"
37
+ ],
38
+ "target_parameters": null,
39
+ "task_type": "CAUSAL_LM",
40
+ "trainable_token_indices": null,
41
+ "use_bdlora": null,
42
+ "use_dora": false,
43
+ "use_qalora": false,
44
+ "use_rslora": false
45
+ }
adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:82f08a1b0170b4ff98bfda895552d803abba116bcbd3fec655bd1abe8fd65efe
3
- size 7874776
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ee97b607fe99019cd18c73a003080dac64169935b3a22b37400abc8378c2ff20
3
+ size 41963912
checkpoint-1509/README.md ADDED
@@ -0,0 +1,209 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: /kaggle/working/models/base/Salesforce__codegen-350M-multi
3
+ library_name: peft
4
+ pipeline_tag: text-generation
5
+ tags:
6
+ - base_model:adapter:/kaggle/working/models/base/Salesforce__codegen-350M-multi
7
+ - lora
8
+ - sft
9
+ - transformers
10
+ - trl
11
+ ---
12
+
13
+ # Model Card for Model ID
14
+
15
+ <!-- Provide a quick summary of what the model is/does. -->
16
+
17
+
18
+
19
+ ## Model Details
20
+
21
+ ### Model Description
22
+
23
+ <!-- Provide a longer summary of what this model is. -->
24
+
25
+
26
+
27
+ - **Developed by:** [More Information Needed]
28
+ - **Funded by [optional]:** [More Information Needed]
29
+ - **Shared by [optional]:** [More Information Needed]
30
+ - **Model type:** [More Information Needed]
31
+ - **Language(s) (NLP):** [More Information Needed]
32
+ - **License:** [More Information Needed]
33
+ - **Finetuned from model [optional]:** [More Information Needed]
34
+
35
+ ### Model Sources [optional]
36
+
37
+ <!-- Provide the basic links for the model. -->
38
+
39
+ - **Repository:** [More Information Needed]
40
+ - **Paper [optional]:** [More Information Needed]
41
+ - **Demo [optional]:** [More Information Needed]
42
+
43
+ ## Uses
44
+
45
+ <!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
46
+
47
+ ### Direct Use
48
+
49
+ <!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
50
+
51
+ [More Information Needed]
52
+
53
+ ### Downstream Use [optional]
54
+
55
+ <!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
56
+
57
+ [More Information Needed]
58
+
59
+ ### Out-of-Scope Use
60
+
61
+ <!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
62
+
63
+ [More Information Needed]
64
+
65
+ ## Bias, Risks, and Limitations
66
+
67
+ <!-- This section is meant to convey both technical and sociotechnical limitations. -->
68
+
69
+ [More Information Needed]
70
+
71
+ ### Recommendations
72
+
73
+ <!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
74
+
75
+ Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
76
+
77
+ ## How to Get Started with the Model
78
+
79
+ Use the code below to get started with the model.
80
+
81
+ [More Information Needed]
82
+
83
+ ## Training Details
84
+
85
+ ### Training Data
86
+
87
+ <!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
88
+
89
+ [More Information Needed]
90
+
91
+ ### Training Procedure
92
+
93
+ <!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
94
+
95
+ #### Preprocessing [optional]
96
+
97
+ [More Information Needed]
98
+
99
+
100
+ #### Training Hyperparameters
101
+
102
+ - **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
103
+
104
+ #### Speeds, Sizes, Times [optional]
105
+
106
+ <!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
107
+
108
+ [More Information Needed]
109
+
110
+ ## Evaluation
111
+
112
+ <!-- This section describes the evaluation protocols and provides the results. -->
113
+
114
+ ### Testing Data, Factors & Metrics
115
+
116
+ #### Testing Data
117
+
118
+ <!-- This should link to a Dataset Card if possible. -->
119
+
120
+ [More Information Needed]
121
+
122
+ #### Factors
123
+
124
+ <!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
125
+
126
+ [More Information Needed]
127
+
128
+ #### Metrics
129
+
130
+ <!-- These are the evaluation metrics being used, ideally with a description of why. -->
131
+
132
+ [More Information Needed]
133
+
134
+ ### Results
135
+
136
+ [More Information Needed]
137
+
138
+ #### Summary
139
+
140
+
141
+
142
+ ## Model Examination [optional]
143
+
144
+ <!-- Relevant interpretability work for the model goes here -->
145
+
146
+ [More Information Needed]
147
+
148
+ ## Environmental Impact
149
+
150
+ <!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
151
+
152
+ Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
153
+
154
+ - **Hardware Type:** [More Information Needed]
155
+ - **Hours used:** [More Information Needed]
156
+ - **Cloud Provider:** [More Information Needed]
157
+ - **Compute Region:** [More Information Needed]
158
+ - **Carbon Emitted:** [More Information Needed]
159
+
160
+ ## Technical Specifications [optional]
161
+
162
+ ### Model Architecture and Objective
163
+
164
+ [More Information Needed]
165
+
166
+ ### Compute Infrastructure
167
+
168
+ [More Information Needed]
169
+
170
+ #### Hardware
171
+
172
+ [More Information Needed]
173
+
174
+ #### Software
175
+
176
+ [More Information Needed]
177
+
178
+ ## Citation [optional]
179
+
180
+ <!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
181
+
182
+ **BibTeX:**
183
+
184
+ [More Information Needed]
185
+
186
+ **APA:**
187
+
188
+ [More Information Needed]
189
+
190
+ ## Glossary [optional]
191
+
192
+ <!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
193
+
194
+ [More Information Needed]
195
+
196
+ ## More Information [optional]
197
+
198
+ [More Information Needed]
199
+
200
+ ## Model Card Authors [optional]
201
+
202
+ [More Information Needed]
203
+
204
+ ## Model Card Contact
205
+
206
+ [More Information Needed]
207
+ ### Framework versions
208
+
209
+ - PEFT 0.19.1
checkpoint-1509/adapter_config.json ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alora_invocation_tokens": null,
3
+ "alpha_pattern": {},
4
+ "arrow_config": null,
5
+ "auto_mapping": null,
6
+ "base_model_name_or_path": "/kaggle/working/models/base/Salesforce__codegen-350M-multi",
7
+ "bias": "none",
8
+ "corda_config": null,
9
+ "ensure_weight_tying": false,
10
+ "eva_config": null,
11
+ "exclude_modules": null,
12
+ "fan_in_fan_out": false,
13
+ "inference_mode": true,
14
+ "init_lora_weights": true,
15
+ "layer_replication": null,
16
+ "layers_pattern": null,
17
+ "layers_to_transform": null,
18
+ "loftq_config": {},
19
+ "lora_alpha": 64,
20
+ "lora_bias": false,
21
+ "lora_dropout": 0.05,
22
+ "lora_ga_config": null,
23
+ "megatron_config": null,
24
+ "megatron_core": "megatron.core",
25
+ "modules_to_save": null,
26
+ "peft_type": "LORA",
27
+ "peft_version": "0.19.1",
28
+ "qalora_group_size": 16,
29
+ "r": 32,
30
+ "rank_pattern": {},
31
+ "revision": null,
32
+ "target_modules": [
33
+ "qkv_proj",
34
+ "out_proj",
35
+ "fc_in",
36
+ "fc_out"
37
+ ],
38
+ "target_parameters": null,
39
+ "task_type": "CAUSAL_LM",
40
+ "trainable_token_indices": null,
41
+ "use_bdlora": null,
42
+ "use_dora": false,
43
+ "use_qalora": false,
44
+ "use_rslora": false
45
+ }
checkpoint-1509/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ee97b607fe99019cd18c73a003080dac64169935b3a22b37400abc8378c2ff20
3
+ size 41963912
checkpoint-1509/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9535db1a16c7675dbefad6ec6ed0a74fec610b7fcf805576a445be7112bc0814
3
+ size 84023435
checkpoint-1509/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a103ee63da4304e0c3e6bf8c37e9987caf868f0c2dcdb6d3aaca72ff4654fd45
3
+ size 14645
checkpoint-1509/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6c94347d64dc2235b6035d81d233c519c36c683c0ab9aeada7031f1f155111b0
3
+ size 1465
checkpoint-1509/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
checkpoint-1509/tokenizer_config.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<|endoftext|>",
5
+ "eos_token": "<|endoftext|>",
6
+ "is_local": true,
7
+ "model_max_length": 2048,
8
+ "pad_token": "<|endoftext|>",
9
+ "tokenizer_class": "TokenizersBackend",
10
+ "unk_token": "<|endoftext|>"
11
+ }
checkpoint-1509/trainer_state.json ADDED
@@ -0,0 +1,1576 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": 1509,
3
+ "best_metric": 1.013381004333496,
4
+ "best_model_checkpoint": "/kaggle/working/models/checkpoints/v4/nosql2doc/checkpoint-1509",
5
+ "epoch": 3.0,
6
+ "eval_steps": 500,
7
+ "global_step": 1509,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "entropy": 2.8841454654932024,
14
+ "epoch": 0.01990049751243781,
15
+ "grad_norm": 3.4684948921203613,
16
+ "learning_rate": 7.142857142857143e-06,
17
+ "loss": 3.3999290466308594,
18
+ "mean_token_accuracy": 0.36815835647284983,
19
+ "num_tokens": 84042.0,
20
+ "step": 10
21
+ },
22
+ {
23
+ "entropy": 2.871115285158157,
24
+ "epoch": 0.03980099502487562,
25
+ "grad_norm": 3.199768304824829,
26
+ "learning_rate": 1.5079365079365079e-05,
27
+ "loss": 3.1389333724975588,
28
+ "mean_token_accuracy": 0.41857333555817605,
29
+ "num_tokens": 166453.0,
30
+ "step": 20
31
+ },
32
+ {
33
+ "entropy": 2.689709846675396,
34
+ "epoch": 0.05970149253731343,
35
+ "grad_norm": 2.241548776626587,
36
+ "learning_rate": 2.3015873015873015e-05,
37
+ "loss": 2.5403181076049806,
38
+ "mean_token_accuracy": 0.45860434919595716,
39
+ "num_tokens": 248357.0,
40
+ "step": 30
41
+ },
42
+ {
43
+ "entropy": 2.371776354312897,
44
+ "epoch": 0.07960199004975124,
45
+ "grad_norm": 2.2443745136260986,
46
+ "learning_rate": 3.095238095238095e-05,
47
+ "loss": 2.2156551361083983,
48
+ "mean_token_accuracy": 0.534111600741744,
49
+ "num_tokens": 329302.0,
50
+ "step": 40
51
+ },
52
+ {
53
+ "entropy": 1.9646751508116722,
54
+ "epoch": 0.09950248756218906,
55
+ "grad_norm": 2.8479607105255127,
56
+ "learning_rate": 3.888888888888889e-05,
57
+ "loss": 1.9395181655883789,
58
+ "mean_token_accuracy": 0.5719782687723637,
59
+ "num_tokens": 410388.0,
60
+ "step": 50
61
+ },
62
+ {
63
+ "entropy": 1.712027932703495,
64
+ "epoch": 0.11940298507462686,
65
+ "grad_norm": 2.5392374992370605,
66
+ "learning_rate": 4.682539682539683e-05,
67
+ "loss": 1.7669334411621094,
68
+ "mean_token_accuracy": 0.5962413489818573,
69
+ "num_tokens": 491142.0,
70
+ "step": 60
71
+ },
72
+ {
73
+ "entropy": 1.6189243733882903,
74
+ "epoch": 0.13930348258706468,
75
+ "grad_norm": 2.8646080493927,
76
+ "learning_rate": 5.4761904761904766e-05,
77
+ "loss": 1.6198507308959962,
78
+ "mean_token_accuracy": 0.6142528053373099,
79
+ "num_tokens": 573054.0,
80
+ "step": 70
81
+ },
82
+ {
83
+ "entropy": 1.6242131367325783,
84
+ "epoch": 0.15920398009950248,
85
+ "grad_norm": 3.0876874923706055,
86
+ "learning_rate": 6.26984126984127e-05,
87
+ "loss": 1.6246612548828125,
88
+ "mean_token_accuracy": 0.626776534691453,
89
+ "num_tokens": 658316.0,
90
+ "step": 80
91
+ },
92
+ {
93
+ "entropy": 1.5123998656868936,
94
+ "epoch": 0.1791044776119403,
95
+ "grad_norm": 2.827972650527954,
96
+ "learning_rate": 7.063492063492065e-05,
97
+ "loss": 1.5039705276489257,
98
+ "mean_token_accuracy": 0.6451249852776527,
99
+ "num_tokens": 738196.0,
100
+ "step": 90
101
+ },
102
+ {
103
+ "entropy": 1.4389337845146657,
104
+ "epoch": 0.19900497512437812,
105
+ "grad_norm": 2.794956684112549,
106
+ "learning_rate": 7.857142857142858e-05,
107
+ "loss": 1.4707328796386718,
108
+ "mean_token_accuracy": 0.6486270293593407,
109
+ "num_tokens": 818366.0,
110
+ "step": 100
111
+ },
112
+ {
113
+ "entropy": 1.4290485948324203,
114
+ "epoch": 0.21890547263681592,
115
+ "grad_norm": 2.650611639022827,
116
+ "learning_rate": 8.650793650793651e-05,
117
+ "loss": 1.461697483062744,
118
+ "mean_token_accuracy": 0.6452589213848114,
119
+ "num_tokens": 900542.0,
120
+ "step": 110
121
+ },
122
+ {
123
+ "entropy": 1.4537153281271458,
124
+ "epoch": 0.23880597014925373,
125
+ "grad_norm": 3.212292194366455,
126
+ "learning_rate": 9.444444444444444e-05,
127
+ "loss": 1.3799741744995118,
128
+ "mean_token_accuracy": 0.665483495220542,
129
+ "num_tokens": 983783.0,
130
+ "step": 120
131
+ },
132
+ {
133
+ "entropy": 1.3213311471045017,
134
+ "epoch": 0.25870646766169153,
135
+ "grad_norm": 2.367784261703491,
136
+ "learning_rate": 0.00010238095238095237,
137
+ "loss": 1.339876365661621,
138
+ "mean_token_accuracy": 0.6738482430577278,
139
+ "num_tokens": 1065282.0,
140
+ "step": 130
141
+ },
142
+ {
143
+ "entropy": 1.2877162352204323,
144
+ "epoch": 0.27860696517412936,
145
+ "grad_norm": 2.7096056938171387,
146
+ "learning_rate": 0.00011031746031746033,
147
+ "loss": 1.3189430236816406,
148
+ "mean_token_accuracy": 0.6620625633746385,
149
+ "num_tokens": 1147430.0,
150
+ "step": 140
151
+ },
152
+ {
153
+ "entropy": 1.3201022632420063,
154
+ "epoch": 0.29850746268656714,
155
+ "grad_norm": 2.497452974319458,
156
+ "learning_rate": 0.00011825396825396826,
157
+ "loss": 1.3049938201904296,
158
+ "mean_token_accuracy": 0.6833859849721193,
159
+ "num_tokens": 1225840.0,
160
+ "step": 150
161
+ },
162
+ {
163
+ "entropy": 1.2007936105132102,
164
+ "epoch": 0.31840796019900497,
165
+ "grad_norm": 3.0387496948242188,
166
+ "learning_rate": 0.0001261904761904762,
167
+ "loss": 1.231965923309326,
168
+ "mean_token_accuracy": 0.7017441529780626,
169
+ "num_tokens": 1309268.0,
170
+ "step": 160
171
+ },
172
+ {
173
+ "entropy": 1.3119081839919091,
174
+ "epoch": 0.3383084577114428,
175
+ "grad_norm": 2.327012300491333,
176
+ "learning_rate": 0.00013412698412698412,
177
+ "loss": 1.2840826034545898,
178
+ "mean_token_accuracy": 0.6792996764183045,
179
+ "num_tokens": 1389094.0,
180
+ "step": 170
181
+ },
182
+ {
183
+ "entropy": 1.2995639897882938,
184
+ "epoch": 0.3582089552238806,
185
+ "grad_norm": 2.426831007003784,
186
+ "learning_rate": 0.00014206349206349208,
187
+ "loss": 1.2992033958435059,
188
+ "mean_token_accuracy": 0.6910909987986088,
189
+ "num_tokens": 1471243.0,
190
+ "step": 180
191
+ },
192
+ {
193
+ "entropy": 1.1952705256640912,
194
+ "epoch": 0.3781094527363184,
195
+ "grad_norm": 2.783355951309204,
196
+ "learning_rate": 0.00015000000000000001,
197
+ "loss": 1.1649029731750489,
198
+ "mean_token_accuracy": 0.7020247872918844,
199
+ "num_tokens": 1552783.0,
200
+ "step": 190
201
+ },
202
+ {
203
+ "entropy": 1.2681411176919937,
204
+ "epoch": 0.39800995024875624,
205
+ "grad_norm": 2.162163019180298,
206
+ "learning_rate": 0.00015793650793650795,
207
+ "loss": 1.2627052307128905,
208
+ "mean_token_accuracy": 0.6855453528463841,
209
+ "num_tokens": 1635174.0,
210
+ "step": 200
211
+ },
212
+ {
213
+ "entropy": 1.2593041777610778,
214
+ "epoch": 0.417910447761194,
215
+ "grad_norm": 2.3045036792755127,
216
+ "learning_rate": 0.0001658730158730159,
217
+ "loss": 1.237012004852295,
218
+ "mean_token_accuracy": 0.6885611288249492,
219
+ "num_tokens": 1720930.0,
220
+ "step": 210
221
+ },
222
+ {
223
+ "entropy": 1.1904637023806572,
224
+ "epoch": 0.43781094527363185,
225
+ "grad_norm": 2.291395664215088,
226
+ "learning_rate": 0.00017380952380952383,
227
+ "loss": 1.2200750350952148,
228
+ "mean_token_accuracy": 0.6859784577041864,
229
+ "num_tokens": 1803132.0,
230
+ "step": 220
231
+ },
232
+ {
233
+ "entropy": 1.1843627236783505,
234
+ "epoch": 0.4577114427860697,
235
+ "grad_norm": 2.553251266479492,
236
+ "learning_rate": 0.00018174603174603177,
237
+ "loss": 1.2164555549621583,
238
+ "mean_token_accuracy": 0.6943051021546125,
239
+ "num_tokens": 1884825.0,
240
+ "step": 230
241
+ },
242
+ {
243
+ "entropy": 1.1876815035939217,
244
+ "epoch": 0.47761194029850745,
245
+ "grad_norm": 2.403580904006958,
246
+ "learning_rate": 0.0001896825396825397,
247
+ "loss": 1.1573832511901856,
248
+ "mean_token_accuracy": 0.7180207531899214,
249
+ "num_tokens": 1968914.0,
250
+ "step": 240
251
+ },
252
+ {
253
+ "entropy": 1.1310456305742265,
254
+ "epoch": 0.4975124378109453,
255
+ "grad_norm": 2.3557302951812744,
256
+ "learning_rate": 0.00019761904761904763,
257
+ "loss": 1.1767531394958497,
258
+ "mean_token_accuracy": 0.6986724361777306,
259
+ "num_tokens": 2050602.0,
260
+ "step": 250
261
+ },
262
+ {
263
+ "entropy": 1.2115244895219803,
264
+ "epoch": 0.5174129353233831,
265
+ "grad_norm": 2.430337905883789,
266
+ "learning_rate": 0.0001999989408126818,
267
+ "loss": 1.187550926208496,
268
+ "mean_token_accuracy": 0.6943733543157578,
269
+ "num_tokens": 2134774.0,
270
+ "step": 260
271
+ },
272
+ {
273
+ "entropy": 1.1603886000812054,
274
+ "epoch": 0.5373134328358209,
275
+ "grad_norm": 1.848976969718933,
276
+ "learning_rate": 0.0001999937530104439,
277
+ "loss": 1.1683053970336914,
278
+ "mean_token_accuracy": 0.6958795577287674,
279
+ "num_tokens": 2214022.0,
280
+ "step": 270
281
+ },
282
+ {
283
+ "entropy": 1.083036269247532,
284
+ "epoch": 0.5572139303482587,
285
+ "grad_norm": 2.297624111175537,
286
+ "learning_rate": 0.00019998424227267588,
287
+ "loss": 1.087782859802246,
288
+ "mean_token_accuracy": 0.7164598941802979,
289
+ "num_tokens": 2295817.0,
290
+ "step": 280
291
+ },
292
+ {
293
+ "entropy": 1.1467085666954517,
294
+ "epoch": 0.5771144278606966,
295
+ "grad_norm": 2.1926214694976807,
296
+ "learning_rate": 0.00019997040901054646,
297
+ "loss": 1.1336830139160157,
298
+ "mean_token_accuracy": 0.7070459358394146,
299
+ "num_tokens": 2379081.0,
300
+ "step": 290
301
+ },
302
+ {
303
+ "entropy": 1.1092145919799805,
304
+ "epoch": 0.5970149253731343,
305
+ "grad_norm": 2.108574390411377,
306
+ "learning_rate": 0.000199952253822096,
307
+ "loss": 1.104503059387207,
308
+ "mean_token_accuracy": 0.7167623668909073,
309
+ "num_tokens": 2461290.0,
310
+ "step": 300
311
+ },
312
+ {
313
+ "entropy": 1.0762298285961152,
314
+ "epoch": 0.6169154228855721,
315
+ "grad_norm": 1.9235388040542603,
316
+ "learning_rate": 0.00019992977749221064,
317
+ "loss": 1.0758417129516602,
318
+ "mean_token_accuracy": 0.7244217373430729,
319
+ "num_tokens": 2546927.0,
320
+ "step": 310
321
+ },
322
+ {
323
+ "entropy": 1.058315774053335,
324
+ "epoch": 0.6368159203980099,
325
+ "grad_norm": 2.015493154525757,
326
+ "learning_rate": 0.0001999029809925883,
327
+ "loss": 1.0714119911193847,
328
+ "mean_token_accuracy": 0.7185824103653431,
329
+ "num_tokens": 2627151.0,
330
+ "step": 320
331
+ },
332
+ {
333
+ "entropy": 1.040397210419178,
334
+ "epoch": 0.6567164179104478,
335
+ "grad_norm": 1.824608564376831,
336
+ "learning_rate": 0.00019987186548169682,
337
+ "loss": 1.0344210624694825,
338
+ "mean_token_accuracy": 0.7333592250943184,
339
+ "num_tokens": 2711367.0,
340
+ "step": 330
341
+ },
342
+ {
343
+ "entropy": 0.9891116008162498,
344
+ "epoch": 0.6766169154228856,
345
+ "grad_norm": 1.9432896375656128,
346
+ "learning_rate": 0.0001998364323047236,
347
+ "loss": 0.9733485221862793,
348
+ "mean_token_accuracy": 0.7416356436908245,
349
+ "num_tokens": 2793468.0,
350
+ "step": 340
351
+ },
352
+ {
353
+ "entropy": 1.0004291333258153,
354
+ "epoch": 0.6965174129353234,
355
+ "grad_norm": 1.906457781791687,
356
+ "learning_rate": 0.00019979668299351783,
357
+ "loss": 1.0494978904724122,
358
+ "mean_token_accuracy": 0.7176768533885479,
359
+ "num_tokens": 2875166.0,
360
+ "step": 350
361
+ },
362
+ {
363
+ "entropy": 1.0571318455040455,
364
+ "epoch": 0.7164179104477612,
365
+ "grad_norm": 1.7709777355194092,
366
+ "learning_rate": 0.00019975261926652392,
367
+ "loss": 1.0371734619140625,
368
+ "mean_token_accuracy": 0.7288566574454307,
369
+ "num_tokens": 2959671.0,
370
+ "step": 360
371
+ },
372
+ {
373
+ "entropy": 0.9674378834664822,
374
+ "epoch": 0.736318407960199,
375
+ "grad_norm": 1.896713376045227,
376
+ "learning_rate": 0.00019970424302870739,
377
+ "loss": 0.9970813751220703,
378
+ "mean_token_accuracy": 0.7395003981888294,
379
+ "num_tokens": 3041437.0,
380
+ "step": 370
381
+ },
382
+ {
383
+ "entropy": 1.1270258143544196,
384
+ "epoch": 0.7562189054726368,
385
+ "grad_norm": 1.7242449522018433,
386
+ "learning_rate": 0.00019965155637147243,
387
+ "loss": 1.0752775192260742,
388
+ "mean_token_accuracy": 0.7305668473243714,
389
+ "num_tokens": 3123865.0,
390
+ "step": 380
391
+ },
392
+ {
393
+ "entropy": 0.9315242812037468,
394
+ "epoch": 0.7761194029850746,
395
+ "grad_norm": 1.781772255897522,
396
+ "learning_rate": 0.0001995945615725716,
397
+ "loss": 0.9751872062683106,
398
+ "mean_token_accuracy": 0.7497406356036663,
399
+ "num_tokens": 3205032.0,
400
+ "step": 390
401
+ },
402
+ {
403
+ "entropy": 1.076903734356165,
404
+ "epoch": 0.7960199004975125,
405
+ "grad_norm": 1.8422024250030518,
406
+ "learning_rate": 0.00019953326109600728,
407
+ "loss": 1.0796968460083007,
408
+ "mean_token_accuracy": 0.7176821112632752,
409
+ "num_tokens": 3286003.0,
410
+ "step": 400
411
+ },
412
+ {
413
+ "entropy": 0.9634652934968472,
414
+ "epoch": 0.8159203980099502,
415
+ "grad_norm": 1.7082111835479736,
416
+ "learning_rate": 0.00019946765759192497,
417
+ "loss": 0.9697890281677246,
418
+ "mean_token_accuracy": 0.7406083434820175,
419
+ "num_tokens": 3367484.0,
420
+ "step": 410
421
+ },
422
+ {
423
+ "entropy": 0.9321402795612812,
424
+ "epoch": 0.835820895522388,
425
+ "grad_norm": 1.8066761493682861,
426
+ "learning_rate": 0.00019939775389649916,
427
+ "loss": 0.9277518272399903,
428
+ "mean_token_accuracy": 0.7574292354285717,
429
+ "num_tokens": 3453904.0,
430
+ "step": 420
431
+ },
432
+ {
433
+ "entropy": 0.9754443395882845,
434
+ "epoch": 0.8557213930348259,
435
+ "grad_norm": 1.6425416469573975,
436
+ "learning_rate": 0.00019932355303181026,
437
+ "loss": 0.9705208778381348,
438
+ "mean_token_accuracy": 0.7435099229216575,
439
+ "num_tokens": 3536472.0,
440
+ "step": 430
441
+ },
442
+ {
443
+ "entropy": 0.9742012087255716,
444
+ "epoch": 0.8756218905472637,
445
+ "grad_norm": 1.707189679145813,
446
+ "learning_rate": 0.00019924505820571425,
447
+ "loss": 1.036739444732666,
448
+ "mean_token_accuracy": 0.7312861289829016,
449
+ "num_tokens": 3616711.0,
450
+ "step": 440
451
+ },
452
+ {
453
+ "entropy": 1.0537842728197575,
454
+ "epoch": 0.8955223880597015,
455
+ "grad_norm": 1.865692377090454,
456
+ "learning_rate": 0.0001991622728117038,
457
+ "loss": 1.0217846870422362,
458
+ "mean_token_accuracy": 0.7338800489902496,
459
+ "num_tokens": 3701126.0,
460
+ "step": 450
461
+ },
462
+ {
463
+ "entropy": 0.989534855633974,
464
+ "epoch": 0.9154228855721394,
465
+ "grad_norm": 1.7353651523590088,
466
+ "learning_rate": 0.00019907520042876172,
467
+ "loss": 1.0003718376159667,
468
+ "mean_token_accuracy": 0.7480997510254384,
469
+ "num_tokens": 3783495.0,
470
+ "step": 460
471
+ },
472
+ {
473
+ "entropy": 0.9294702004641294,
474
+ "epoch": 0.9353233830845771,
475
+ "grad_norm": 2.007262706756592,
476
+ "learning_rate": 0.00019898384482120614,
477
+ "loss": 1.0100757598876953,
478
+ "mean_token_accuracy": 0.7364842720329762,
479
+ "num_tokens": 3863784.0,
480
+ "step": 470
481
+ },
482
+ {
483
+ "entropy": 0.9523782070726157,
484
+ "epoch": 0.9552238805970149,
485
+ "grad_norm": 1.9396175146102905,
486
+ "learning_rate": 0.0001988882099385278,
487
+ "loss": 0.9432971000671386,
488
+ "mean_token_accuracy": 0.756897260248661,
489
+ "num_tokens": 3947540.0,
490
+ "step": 480
491
+ },
492
+ {
493
+ "entropy": 0.9421051684767008,
494
+ "epoch": 0.9751243781094527,
495
+ "grad_norm": 1.8582719564437866,
496
+ "learning_rate": 0.00019878829991521935,
497
+ "loss": 0.9125359535217286,
498
+ "mean_token_accuracy": 0.7575220972299576,
499
+ "num_tokens": 4031284.0,
500
+ "step": 490
501
+ },
502
+ {
503
+ "entropy": 0.9381607126444578,
504
+ "epoch": 0.9950248756218906,
505
+ "grad_norm": 1.9623899459838867,
506
+ "learning_rate": 0.00019868411907059645,
507
+ "loss": 0.9656861305236817,
508
+ "mean_token_accuracy": 0.7544871643185616,
509
+ "num_tokens": 4112149.0,
510
+ "step": 500
511
+ },
512
+ {
513
+ "epoch": 1.0,
514
+ "eval_entropy": 0.9874700385516214,
515
+ "eval_loss": 1.063795566558838,
516
+ "eval_mean_token_accuracy": 0.7334998920847551,
517
+ "eval_num_tokens": 4132656.0,
518
+ "eval_runtime": 88.6229,
519
+ "eval_samples_per_second": 11.679,
520
+ "eval_steps_per_second": 5.845,
521
+ "step": 503
522
+ },
523
+ {
524
+ "entropy": 0.8532990099568116,
525
+ "epoch": 1.0139303482587065,
526
+ "grad_norm": 1.6549201011657715,
527
+ "learning_rate": 0.00019857567190861126,
528
+ "loss": 0.7886298179626465,
529
+ "mean_token_accuracy": 0.7860889779893976,
530
+ "num_tokens": 4190723.0,
531
+ "step": 510
532
+ },
533
+ {
534
+ "entropy": 0.7671804554760456,
535
+ "epoch": 1.0338308457711443,
536
+ "grad_norm": 1.6148929595947266,
537
+ "learning_rate": 0.00019846296311765754,
538
+ "loss": 0.777919864654541,
539
+ "mean_token_accuracy": 0.789631099998951,
540
+ "num_tokens": 4273358.0,
541
+ "step": 520
542
+ },
543
+ {
544
+ "entropy": 0.8041674952954054,
545
+ "epoch": 1.053731343283582,
546
+ "grad_norm": 1.765950322151184,
547
+ "learning_rate": 0.00019834599757036803,
548
+ "loss": 0.7967178821563721,
549
+ "mean_token_accuracy": 0.784614659100771,
550
+ "num_tokens": 4356030.0,
551
+ "step": 530
552
+ },
553
+ {
554
+ "entropy": 0.7405303593724966,
555
+ "epoch": 1.07363184079602,
556
+ "grad_norm": 1.7868931293487549,
557
+ "learning_rate": 0.00019822478032340387,
558
+ "loss": 0.7301093578338623,
559
+ "mean_token_accuracy": 0.8020888641476631,
560
+ "num_tokens": 4440157.0,
561
+ "step": 540
562
+ },
563
+ {
564
+ "entropy": 0.7550990927964449,
565
+ "epoch": 1.0935323383084576,
566
+ "grad_norm": 1.991782784461975,
567
+ "learning_rate": 0.0001980993166172358,
568
+ "loss": 0.7884031295776367,
569
+ "mean_token_accuracy": 0.7879343688488006,
570
+ "num_tokens": 4523343.0,
571
+ "step": 550
572
+ },
573
+ {
574
+ "entropy": 0.812371177971363,
575
+ "epoch": 1.1134328358208956,
576
+ "grad_norm": 1.869396686553955,
577
+ "learning_rate": 0.00019796961187591781,
578
+ "loss": 0.8159590721130371,
579
+ "mean_token_accuracy": 0.7711428456008435,
580
+ "num_tokens": 4605454.0,
581
+ "step": 560
582
+ },
583
+ {
584
+ "entropy": 0.7879433136433363,
585
+ "epoch": 1.1333333333333333,
586
+ "grad_norm": 1.9530060291290283,
587
+ "learning_rate": 0.00019783567170685262,
588
+ "loss": 0.8079433441162109,
589
+ "mean_token_accuracy": 0.7875082984566688,
590
+ "num_tokens": 4687318.0,
591
+ "step": 570
592
+ },
593
+ {
594
+ "entropy": 0.8396137218922377,
595
+ "epoch": 1.153233830845771,
596
+ "grad_norm": 1.7895166873931885,
597
+ "learning_rate": 0.00019769750190054905,
598
+ "loss": 0.8188363075256347,
599
+ "mean_token_accuracy": 0.7710263684391976,
600
+ "num_tokens": 4768232.0,
601
+ "step": 580
602
+ },
603
+ {
604
+ "entropy": 0.8406378872692585,
605
+ "epoch": 1.173134328358209,
606
+ "grad_norm": 1.8147255182266235,
607
+ "learning_rate": 0.00019755510843037191,
608
+ "loss": 0.8532954216003418,
609
+ "mean_token_accuracy": 0.7680545568466186,
610
+ "num_tokens": 4850366.0,
611
+ "step": 590
612
+ },
613
+ {
614
+ "entropy": 0.7758986987173557,
615
+ "epoch": 1.1930348258706467,
616
+ "grad_norm": 1.9839149713516235,
617
+ "learning_rate": 0.00019740849745228367,
618
+ "loss": 0.7972042560577393,
619
+ "mean_token_accuracy": 0.77668926268816,
620
+ "num_tokens": 4932940.0,
621
+ "step": 600
622
+ },
623
+ {
624
+ "entropy": 0.825455692410469,
625
+ "epoch": 1.2129353233830846,
626
+ "grad_norm": 1.776328444480896,
627
+ "learning_rate": 0.00019725767530457837,
628
+ "loss": 0.7664913654327392,
629
+ "mean_token_accuracy": 0.7940710753202438,
630
+ "num_tokens": 5012821.0,
631
+ "step": 610
632
+ },
633
+ {
634
+ "entropy": 0.7286036755889654,
635
+ "epoch": 1.2328358208955223,
636
+ "grad_norm": 2.0241360664367676,
637
+ "learning_rate": 0.00019710264850760756,
638
+ "loss": 0.7731894969940185,
639
+ "mean_token_accuracy": 0.7846131704747676,
640
+ "num_tokens": 5095340.0,
641
+ "step": 620
642
+ },
643
+ {
644
+ "entropy": 0.7920811083167791,
645
+ "epoch": 1.25273631840796,
646
+ "grad_norm": 1.7310129404067993,
647
+ "learning_rate": 0.00019694342376349835,
648
+ "loss": 0.7303972244262695,
649
+ "mean_token_accuracy": 0.7983584970235824,
650
+ "num_tokens": 5177547.0,
651
+ "step": 630
652
+ },
653
+ {
654
+ "entropy": 0.7178942065685987,
655
+ "epoch": 1.272636815920398,
656
+ "grad_norm": 1.7608613967895508,
657
+ "learning_rate": 0.00019678000795586386,
658
+ "loss": 0.8018975257873535,
659
+ "mean_token_accuracy": 0.7767767131328582,
660
+ "num_tokens": 5265612.0,
661
+ "step": 640
662
+ },
663
+ {
664
+ "entropy": 0.8001765824854374,
665
+ "epoch": 1.292537313432836,
666
+ "grad_norm": 2.0576155185699463,
667
+ "learning_rate": 0.00019661240814950536,
668
+ "loss": 0.7223423480987549,
669
+ "mean_token_accuracy": 0.7918425254523754,
670
+ "num_tokens": 5348652.0,
671
+ "step": 650
672
+ },
673
+ {
674
+ "entropy": 0.7580439992249012,
675
+ "epoch": 1.3124378109452737,
676
+ "grad_norm": 1.539169430732727,
677
+ "learning_rate": 0.00019644063159010716,
678
+ "loss": 0.8217278480529785,
679
+ "mean_token_accuracy": 0.7793662197887897,
680
+ "num_tokens": 5431834.0,
681
+ "step": 660
682
+ },
683
+ {
684
+ "entropy": 0.8075141672044992,
685
+ "epoch": 1.3323383084577114,
686
+ "grad_norm": 1.8933053016662598,
687
+ "learning_rate": 0.00019626468570392298,
688
+ "loss": 0.7759135723114013,
689
+ "mean_token_accuracy": 0.7861333280801773,
690
+ "num_tokens": 5514228.0,
691
+ "step": 670
692
+ },
693
+ {
694
+ "entropy": 0.7623774103820324,
695
+ "epoch": 1.3522388059701491,
696
+ "grad_norm": 1.7051018476486206,
697
+ "learning_rate": 0.00019608457809745537,
698
+ "loss": 0.8088616371154785,
699
+ "mean_token_accuracy": 0.7746041730046272,
700
+ "num_tokens": 5594702.0,
701
+ "step": 680
702
+ },
703
+ {
704
+ "entropy": 0.8089807592332363,
705
+ "epoch": 1.372139303482587,
706
+ "grad_norm": 1.8061248064041138,
707
+ "learning_rate": 0.00019590031655712631,
708
+ "loss": 0.7946909904479981,
709
+ "mean_token_accuracy": 0.7781320951879025,
710
+ "num_tokens": 5676309.0,
711
+ "step": 690
712
+ },
713
+ {
714
+ "entropy": 0.7627177253365517,
715
+ "epoch": 1.392039800995025,
716
+ "grad_norm": 1.9005513191223145,
717
+ "learning_rate": 0.00019571190904894115,
718
+ "loss": 0.7615675926208496,
719
+ "mean_token_accuracy": 0.788880155980587,
720
+ "num_tokens": 5757314.0,
721
+ "step": 700
722
+ },
723
+ {
724
+ "entropy": 0.7580569207668304,
725
+ "epoch": 1.4119402985074627,
726
+ "grad_norm": 1.9883071184158325,
727
+ "learning_rate": 0.00019551936371814375,
728
+ "loss": 0.7611291408538818,
729
+ "mean_token_accuracy": 0.7861187107861042,
730
+ "num_tokens": 5838877.0,
731
+ "step": 710
732
+ },
733
+ {
734
+ "entropy": 0.7182941000908614,
735
+ "epoch": 1.4318407960199004,
736
+ "grad_norm": 2.187103033065796,
737
+ "learning_rate": 0.0001953226888888647,
738
+ "loss": 0.7204749584197998,
739
+ "mean_token_accuracy": 0.7946790263056756,
740
+ "num_tokens": 5920820.0,
741
+ "step": 720
742
+ },
743
+ {
744
+ "entropy": 0.6957223568111658,
745
+ "epoch": 1.4517412935323384,
746
+ "grad_norm": 1.7006845474243164,
747
+ "learning_rate": 0.00019512189306376118,
748
+ "loss": 0.757196044921875,
749
+ "mean_token_accuracy": 0.7921811021864414,
750
+ "num_tokens": 6004382.0,
751
+ "step": 730
752
+ },
753
+ {
754
+ "entropy": 0.7850441809743642,
755
+ "epoch": 1.471641791044776,
756
+ "grad_norm": 1.6335111856460571,
757
+ "learning_rate": 0.0001949169849236496,
758
+ "loss": 0.7292089462280273,
759
+ "mean_token_accuracy": 0.797366950660944,
760
+ "num_tokens": 6086449.0,
761
+ "step": 740
762
+ },
763
+ {
764
+ "entropy": 0.7639980979263783,
765
+ "epoch": 1.491542288557214,
766
+ "grad_norm": 1.7224810123443604,
767
+ "learning_rate": 0.00019470797332713012,
768
+ "loss": 0.8179457664489747,
769
+ "mean_token_accuracy": 0.7910384394228458,
770
+ "num_tokens": 6168749.0,
771
+ "step": 750
772
+ },
773
+ {
774
+ "entropy": 0.8024922411888837,
775
+ "epoch": 1.5114427860696518,
776
+ "grad_norm": 1.9208451509475708,
777
+ "learning_rate": 0.0001944948673102038,
778
+ "loss": 0.7916707515716552,
779
+ "mean_token_accuracy": 0.7822470977902413,
780
+ "num_tokens": 6248672.0,
781
+ "step": 760
782
+ },
783
+ {
784
+ "entropy": 0.7430106606334448,
785
+ "epoch": 1.5313432835820895,
786
+ "grad_norm": 1.9618816375732422,
787
+ "learning_rate": 0.00019427767608588183,
788
+ "loss": 0.7538708209991455,
789
+ "mean_token_accuracy": 0.7816703900694847,
790
+ "num_tokens": 6331261.0,
791
+ "step": 770
792
+ },
793
+ {
794
+ "entropy": 0.7457086000591516,
795
+ "epoch": 1.5512437810945272,
796
+ "grad_norm": 1.6858505010604858,
797
+ "learning_rate": 0.0001940564090437875,
798
+ "loss": 0.7316685676574707,
799
+ "mean_token_accuracy": 0.8001063913106918,
800
+ "num_tokens": 6415504.0,
801
+ "step": 780
802
+ },
803
+ {
804
+ "entropy": 0.7096565395593644,
805
+ "epoch": 1.5711442786069652,
806
+ "grad_norm": 1.9956564903259277,
807
+ "learning_rate": 0.00019383107574974984,
808
+ "loss": 0.7277198791503906,
809
+ "mean_token_accuracy": 0.7895878478884697,
810
+ "num_tokens": 6497199.0,
811
+ "step": 790
812
+ },
813
+ {
814
+ "entropy": 0.7606754396110773,
815
+ "epoch": 1.591044776119403,
816
+ "grad_norm": 1.7207188606262207,
817
+ "learning_rate": 0.00019360168594539055,
818
+ "loss": 0.7483164310455322,
819
+ "mean_token_accuracy": 0.7934170700609684,
820
+ "num_tokens": 6577494.0,
821
+ "step": 800
822
+ },
823
+ {
824
+ "entropy": 0.7786010228097439,
825
+ "epoch": 1.6109452736318408,
826
+ "grad_norm": 1.7885297536849976,
827
+ "learning_rate": 0.0001933682495477024,
828
+ "loss": 0.7517632007598877,
829
+ "mean_token_accuracy": 0.791710589081049,
830
+ "num_tokens": 6660010.0,
831
+ "step": 810
832
+ },
833
+ {
834
+ "entropy": 0.73126558996737,
835
+ "epoch": 1.6308457711442785,
836
+ "grad_norm": 1.929076910018921,
837
+ "learning_rate": 0.00019313077664862092,
838
+ "loss": 0.7747976303100585,
839
+ "mean_token_accuracy": 0.7895058654248714,
840
+ "num_tokens": 6739669.0,
841
+ "step": 820
842
+ },
843
+ {
844
+ "entropy": 0.7332248043268919,
845
+ "epoch": 1.6507462686567163,
846
+ "grad_norm": 1.6692851781845093,
847
+ "learning_rate": 0.00019288927751458766,
848
+ "loss": 0.6981019020080567,
849
+ "mean_token_accuracy": 0.8055072821676731,
850
+ "num_tokens": 6819838.0,
851
+ "step": 830
852
+ },
853
+ {
854
+ "entropy": 0.712925647944212,
855
+ "epoch": 1.6706467661691542,
856
+ "grad_norm": 1.5395867824554443,
857
+ "learning_rate": 0.0001926437625861068,
858
+ "loss": 0.7524178504943848,
859
+ "mean_token_accuracy": 0.7908283203840256,
860
+ "num_tokens": 6901777.0,
861
+ "step": 840
862
+ },
863
+ {
864
+ "entropy": 0.7585150092840195,
865
+ "epoch": 1.6905472636815921,
866
+ "grad_norm": 1.6641769409179688,
867
+ "learning_rate": 0.00019239424247729345,
868
+ "loss": 0.7306118488311768,
869
+ "mean_token_accuracy": 0.7910356394946575,
870
+ "num_tokens": 6983222.0,
871
+ "step": 850
872
+ },
873
+ {
874
+ "entropy": 0.7012663371860981,
875
+ "epoch": 1.7104477611940299,
876
+ "grad_norm": 1.9537489414215088,
877
+ "learning_rate": 0.0001921407279754149,
878
+ "loss": 0.752435302734375,
879
+ "mean_token_accuracy": 0.8054570883512497,
880
+ "num_tokens": 7067291.0,
881
+ "step": 860
882
+ },
883
+ {
884
+ "entropy": 0.7264402851462364,
885
+ "epoch": 1.7303482587064676,
886
+ "grad_norm": 1.8359386920928955,
887
+ "learning_rate": 0.00019188323004042435,
888
+ "loss": 0.6932102680206299,
889
+ "mean_token_accuracy": 0.802193396538496,
890
+ "num_tokens": 7150268.0,
891
+ "step": 870
892
+ },
893
+ {
894
+ "entropy": 0.6838662784546614,
895
+ "epoch": 1.7502487562189055,
896
+ "grad_norm": 1.9146333932876587,
897
+ "learning_rate": 0.00019162175980448688,
898
+ "loss": 0.7538199424743652,
899
+ "mean_token_accuracy": 0.796929395198822,
900
+ "num_tokens": 7231091.0,
901
+ "step": 880
902
+ },
903
+ {
904
+ "entropy": 0.7262898899614811,
905
+ "epoch": 1.7701492537313432,
906
+ "grad_norm": 2.0510952472686768,
907
+ "learning_rate": 0.0001913563285714984,
908
+ "loss": 0.7104074954986572,
909
+ "mean_token_accuracy": 0.8008449010550975,
910
+ "num_tokens": 7313340.0,
911
+ "step": 890
912
+ },
913
+ {
914
+ "entropy": 0.7481670882552862,
915
+ "epoch": 1.7900497512437812,
916
+ "grad_norm": 2.0180599689483643,
917
+ "learning_rate": 0.0001910869478165969,
918
+ "loss": 0.8105827331542969,
919
+ "mean_token_accuracy": 0.7843330048024655,
920
+ "num_tokens": 7393519.0,
921
+ "step": 900
922
+ },
923
+ {
924
+ "entropy": 0.7768452275544405,
925
+ "epoch": 1.809950248756219,
926
+ "grad_norm": 1.646704912185669,
927
+ "learning_rate": 0.00019081362918566618,
928
+ "loss": 0.7446431636810302,
929
+ "mean_token_accuracy": 0.8008080549538136,
930
+ "num_tokens": 7475016.0,
931
+ "step": 910
932
+ },
933
+ {
934
+ "entropy": 0.683229249343276,
935
+ "epoch": 1.8298507462686566,
936
+ "grad_norm": 1.638253092765808,
937
+ "learning_rate": 0.00019053638449483259,
938
+ "loss": 0.7266605854034424,
939
+ "mean_token_accuracy": 0.7936579927802085,
940
+ "num_tokens": 7559243.0,
941
+ "step": 920
942
+ },
943
+ {
944
+ "entropy": 0.7504678606987,
945
+ "epoch": 1.8497512437810946,
946
+ "grad_norm": 1.6552717685699463,
947
+ "learning_rate": 0.0001902552257299542,
948
+ "loss": 0.738245677947998,
949
+ "mean_token_accuracy": 0.7894984453916549,
950
+ "num_tokens": 7643029.0,
951
+ "step": 930
952
+ },
953
+ {
954
+ "entropy": 0.7069665104150772,
955
+ "epoch": 1.8696517412935323,
956
+ "grad_norm": 1.4619513750076294,
957
+ "learning_rate": 0.00018997016504610246,
958
+ "loss": 0.7275553226470948,
959
+ "mean_token_accuracy": 0.8092396840453148,
960
+ "num_tokens": 7726479.0,
961
+ "step": 940
962
+ },
963
+ {
964
+ "entropy": 0.6909922767430544,
965
+ "epoch": 1.8895522388059702,
966
+ "grad_norm": 1.6785398721694946,
967
+ "learning_rate": 0.00018968121476703678,
968
+ "loss": 0.6958948612213135,
969
+ "mean_token_accuracy": 0.8184511929750442,
970
+ "num_tokens": 7808235.0,
971
+ "step": 950
972
+ },
973
+ {
974
+ "entropy": 0.7206986505538225,
975
+ "epoch": 1.909452736318408,
976
+ "grad_norm": 1.8156358003616333,
977
+ "learning_rate": 0.00018938838738467184,
978
+ "loss": 0.7307656288146973,
979
+ "mean_token_accuracy": 0.795404677093029,
980
+ "num_tokens": 7888802.0,
981
+ "step": 960
982
+ },
983
+ {
984
+ "entropy": 0.7154657423496247,
985
+ "epoch": 1.9293532338308457,
986
+ "grad_norm": 1.9570693969726562,
987
+ "learning_rate": 0.00018909169555853743,
988
+ "loss": 0.6951888084411622,
989
+ "mean_token_accuracy": 0.7979383312165738,
990
+ "num_tokens": 7970204.0,
991
+ "step": 970
992
+ },
993
+ {
994
+ "entropy": 0.6446726582944393,
995
+ "epoch": 1.9492537313432836,
996
+ "grad_norm": 1.5071886777877808,
997
+ "learning_rate": 0.00018879115211523117,
998
+ "loss": 0.6544716358184814,
999
+ "mean_token_accuracy": 0.8045676186680794,
1000
+ "num_tokens": 8053307.0,
1001
+ "step": 980
1002
+ },
1003
+ {
1004
+ "entropy": 0.699823185056448,
1005
+ "epoch": 1.9691542288557216,
1006
+ "grad_norm": 1.4266027212142944,
1007
+ "learning_rate": 0.0001884867700478641,
1008
+ "loss": 0.7055366516113282,
1009
+ "mean_token_accuracy": 0.7992964766919612,
1010
+ "num_tokens": 8135193.0,
1011
+ "step": 990
1012
+ },
1013
+ {
1014
+ "entropy": 0.7140736062079668,
1015
+ "epoch": 1.9890547263681593,
1016
+ "grad_norm": 2.0378243923187256,
1017
+ "learning_rate": 0.00018817856251549867,
1018
+ "loss": 0.7184643268585205,
1019
+ "mean_token_accuracy": 0.7989312529563903,
1020
+ "num_tokens": 8219594.0,
1021
+ "step": 1000
1022
+ },
1023
+ {
1024
+ "epoch": 2.0,
1025
+ "eval_entropy": 0.8002531397008988,
1026
+ "eval_loss": 1.0154445171356201,
1027
+ "eval_mean_token_accuracy": 0.7423216365709268,
1028
+ "eval_num_tokens": 8265312.0,
1029
+ "eval_runtime": 88.2,
1030
+ "eval_samples_per_second": 11.735,
1031
+ "eval_steps_per_second": 5.873,
1032
+ "step": 1006
1033
+ },
1034
+ {
1035
+ "entropy": 0.7038449613671554,
1036
+ "epoch": 2.007960199004975,
1037
+ "grad_norm": 1.6040326356887817,
1038
+ "learning_rate": 0.00018786654284258034,
1039
+ "loss": 0.6638185501098632,
1040
+ "mean_token_accuracy": 0.8091712115626586,
1041
+ "num_tokens": 8297623.0,
1042
+ "step": 1010
1043
+ },
1044
+ {
1045
+ "entropy": 0.5252644840627909,
1046
+ "epoch": 2.027860696517413,
1047
+ "grad_norm": 2.1827661991119385,
1048
+ "learning_rate": 0.00018755072451836097,
1049
+ "loss": 0.5153174877166748,
1050
+ "mean_token_accuracy": 0.8446948520839215,
1051
+ "num_tokens": 8379050.0,
1052
+ "step": 1020
1053
+ },
1054
+ {
1055
+ "entropy": 0.5121854435652494,
1056
+ "epoch": 2.047761194029851,
1057
+ "grad_norm": 1.8835452795028687,
1058
+ "learning_rate": 0.00018723112119631608,
1059
+ "loss": 0.5017234325408936,
1060
+ "mean_token_accuracy": 0.8539764225482941,
1061
+ "num_tokens": 8462847.0,
1062
+ "step": 1030
1063
+ },
1064
+ {
1065
+ "entropy": 0.5651770515367389,
1066
+ "epoch": 2.0676616915422885,
1067
+ "grad_norm": 1.6592763662338257,
1068
+ "learning_rate": 0.00018690774669355442,
1069
+ "loss": 0.5333661079406739,
1070
+ "mean_token_accuracy": 0.8419696733355522,
1071
+ "num_tokens": 8542681.0,
1072
+ "step": 1040
1073
+ },
1074
+ {
1075
+ "entropy": 0.47107693143188956,
1076
+ "epoch": 2.0875621890547262,
1077
+ "grad_norm": 1.8748646974563599,
1078
+ "learning_rate": 0.00018658061499022055,
1079
+ "loss": 0.4740941047668457,
1080
+ "mean_token_accuracy": 0.8567564368247986,
1081
+ "num_tokens": 8627432.0,
1082
+ "step": 1050
1083
+ },
1084
+ {
1085
+ "entropy": 0.49270418751984835,
1086
+ "epoch": 2.107462686567164,
1087
+ "grad_norm": 2.027717351913452,
1088
+ "learning_rate": 0.0001862497402288906,
1089
+ "loss": 0.48979806900024414,
1090
+ "mean_token_accuracy": 0.8501696541905404,
1091
+ "num_tokens": 8708822.0,
1092
+ "step": 1060
1093
+ },
1094
+ {
1095
+ "entropy": 0.4634111661463976,
1096
+ "epoch": 2.127363184079602,
1097
+ "grad_norm": 1.8452632427215576,
1098
+ "learning_rate": 0.0001859151367139608,
1099
+ "loss": 0.4521069049835205,
1100
+ "mean_token_accuracy": 0.8683336563408375,
1101
+ "num_tokens": 8789791.0,
1102
+ "step": 1070
1103
+ },
1104
+ {
1105
+ "entropy": 0.46942942440509794,
1106
+ "epoch": 2.14726368159204,
1107
+ "grad_norm": 1.9295680522918701,
1108
+ "learning_rate": 0.000185576818911029,
1109
+ "loss": 0.5178594589233398,
1110
+ "mean_token_accuracy": 0.8462874710559845,
1111
+ "num_tokens": 8870514.0,
1112
+ "step": 1080
1113
+ },
1114
+ {
1115
+ "entropy": 0.5393101962283253,
1116
+ "epoch": 2.1671641791044776,
1117
+ "grad_norm": 1.6308494806289673,
1118
+ "learning_rate": 0.00018523480144626948,
1119
+ "loss": 0.5196755409240723,
1120
+ "mean_token_accuracy": 0.8491615362465381,
1121
+ "num_tokens": 8952466.0,
1122
+ "step": 1090
1123
+ },
1124
+ {
1125
+ "entropy": 0.491584631241858,
1126
+ "epoch": 2.1870646766169153,
1127
+ "grad_norm": 1.9126944541931152,
1128
+ "learning_rate": 0.00018488909910580037,
1129
+ "loss": 0.507460069656372,
1130
+ "mean_token_accuracy": 0.8491294652223587,
1131
+ "num_tokens": 9034965.0,
1132
+ "step": 1100
1133
+ },
1134
+ {
1135
+ "entropy": 0.5460982380434871,
1136
+ "epoch": 2.206965174129353,
1137
+ "grad_norm": 2.245730400085449,
1138
+ "learning_rate": 0.00018453972683504466,
1139
+ "loss": 0.5434001445770263,
1140
+ "mean_token_accuracy": 0.8417311415076256,
1141
+ "num_tokens": 9117636.0,
1142
+ "step": 1110
1143
+ },
1144
+ {
1145
+ "entropy": 0.5413472417742013,
1146
+ "epoch": 2.226865671641791,
1147
+ "grad_norm": 1.7490206956863403,
1148
+ "learning_rate": 0.00018418669973808386,
1149
+ "loss": 0.5176630973815918,
1150
+ "mean_token_accuracy": 0.8437662713229657,
1151
+ "num_tokens": 9203719.0,
1152
+ "step": 1120
1153
+ },
1154
+ {
1155
+ "entropy": 0.5068975739181042,
1156
+ "epoch": 2.246766169154229,
1157
+ "grad_norm": 2.32169246673584,
1158
+ "learning_rate": 0.00018383003307700525,
1159
+ "loss": 0.4966591358184814,
1160
+ "mean_token_accuracy": 0.8558630302548409,
1161
+ "num_tokens": 9285285.0,
1162
+ "step": 1130
1163
+ },
1164
+ {
1165
+ "entropy": 0.472692010179162,
1166
+ "epoch": 2.2666666666666666,
1167
+ "grad_norm": 2.121119499206543,
1168
+ "learning_rate": 0.0001834697422712419,
1169
+ "loss": 0.4562082290649414,
1170
+ "mean_token_accuracy": 0.861832109093666,
1171
+ "num_tokens": 9368724.0,
1172
+ "step": 1140
1173
+ },
1174
+ {
1175
+ "entropy": 0.48463920764625074,
1176
+ "epoch": 2.2865671641791043,
1177
+ "grad_norm": 2.003596305847168,
1178
+ "learning_rate": 0.00018310584289690608,
1179
+ "loss": 0.5065945148468017,
1180
+ "mean_token_accuracy": 0.8411129087209701,
1181
+ "num_tokens": 9450345.0,
1182
+ "step": 1150
1183
+ },
1184
+ {
1185
+ "entropy": 0.5297361209988594,
1186
+ "epoch": 2.306467661691542,
1187
+ "grad_norm": 1.9170504808425903,
1188
+ "learning_rate": 0.0001827383506861159,
1189
+ "loss": 0.5093509674072265,
1190
+ "mean_token_accuracy": 0.8439007833600044,
1191
+ "num_tokens": 9531914.0,
1192
+ "step": 1160
1193
+ },
1194
+ {
1195
+ "entropy": 0.508120141364634,
1196
+ "epoch": 2.32636815920398,
1197
+ "grad_norm": 1.8182140588760376,
1198
+ "learning_rate": 0.00018236728152631526,
1199
+ "loss": 0.5065027236938476,
1200
+ "mean_token_accuracy": 0.8519511334598064,
1201
+ "num_tokens": 9613824.0,
1202
+ "step": 1170
1203
+ },
1204
+ {
1205
+ "entropy": 0.5490097716450691,
1206
+ "epoch": 2.346268656716418,
1207
+ "grad_norm": 1.6451319456100464,
1208
+ "learning_rate": 0.00018199265145958678,
1209
+ "loss": 0.5736330032348633,
1210
+ "mean_token_accuracy": 0.8286732420325279,
1211
+ "num_tokens": 9696070.0,
1212
+ "step": 1180
1213
+ },
1214
+ {
1215
+ "entropy": 0.5155842589214444,
1216
+ "epoch": 2.3661691542288557,
1217
+ "grad_norm": 1.8924293518066406,
1218
+ "learning_rate": 0.00018161447668195858,
1219
+ "loss": 0.5310076236724853,
1220
+ "mean_token_accuracy": 0.8431366585195065,
1221
+ "num_tokens": 9777568.0,
1222
+ "step": 1190
1223
+ },
1224
+ {
1225
+ "entropy": 0.4998965173959732,
1226
+ "epoch": 2.3860696517412934,
1227
+ "grad_norm": 1.9782637357711792,
1228
+ "learning_rate": 0.00018123277354270372,
1229
+ "loss": 0.46693716049194334,
1230
+ "mean_token_accuracy": 0.8578181132674217,
1231
+ "num_tokens": 9858081.0,
1232
+ "step": 1200
1233
+ },
1234
+ {
1235
+ "entropy": 0.5187313890084624,
1236
+ "epoch": 2.405970149253731,
1237
+ "grad_norm": 1.8333914279937744,
1238
+ "learning_rate": 0.00018084755854363377,
1239
+ "loss": 0.5384686946868896,
1240
+ "mean_token_accuracy": 0.8451795615255833,
1241
+ "num_tokens": 9940187.0,
1242
+ "step": 1210
1243
+ },
1244
+ {
1245
+ "entropy": 0.5085030103102326,
1246
+ "epoch": 2.4258706467661693,
1247
+ "grad_norm": 2.00839900970459,
1248
+ "learning_rate": 0.00018045884833838512,
1249
+ "loss": 0.5079335689544677,
1250
+ "mean_token_accuracy": 0.8514300569891929,
1251
+ "num_tokens": 10018432.0,
1252
+ "step": 1220
1253
+ },
1254
+ {
1255
+ "entropy": 0.5382265558466315,
1256
+ "epoch": 2.445771144278607,
1257
+ "grad_norm": 1.9003617763519287,
1258
+ "learning_rate": 0.00018006665973169911,
1259
+ "loss": 0.5386298179626465,
1260
+ "mean_token_accuracy": 0.8454402416944504,
1261
+ "num_tokens": 10100137.0,
1262
+ "step": 1230
1263
+ },
1264
+ {
1265
+ "entropy": 0.5438722034916281,
1266
+ "epoch": 2.4656716417910447,
1267
+ "grad_norm": 2.1470773220062256,
1268
+ "learning_rate": 0.0001796710096786956,
1269
+ "loss": 0.5553887367248536,
1270
+ "mean_token_accuracy": 0.8391343042254448,
1271
+ "num_tokens": 10184025.0,
1272
+ "step": 1240
1273
+ },
1274
+ {
1275
+ "entropy": 0.5251597924157977,
1276
+ "epoch": 2.4855721393034824,
1277
+ "grad_norm": 1.992197036743164,
1278
+ "learning_rate": 0.0001792719152841398,
1279
+ "loss": 0.5232258319854737,
1280
+ "mean_token_accuracy": 0.8480332940816879,
1281
+ "num_tokens": 10268664.0,
1282
+ "step": 1250
1283
+ },
1284
+ {
1285
+ "entropy": 0.5361188564449548,
1286
+ "epoch": 2.50547263681592,
1287
+ "grad_norm": 1.6610969305038452,
1288
+ "learning_rate": 0.0001788693938017029,
1289
+ "loss": 0.5130891799926758,
1290
+ "mean_token_accuracy": 0.8441365607082844,
1291
+ "num_tokens": 10349492.0,
1292
+ "step": 1260
1293
+ },
1294
+ {
1295
+ "entropy": 0.4794240856543183,
1296
+ "epoch": 2.5253731343283583,
1297
+ "grad_norm": 2.0173373222351074,
1298
+ "learning_rate": 0.00017846346263321623,
1299
+ "loss": 0.5028162956237793,
1300
+ "mean_token_accuracy": 0.84928208142519,
1301
+ "num_tokens": 10432597.0,
1302
+ "step": 1270
1303
+ },
1304
+ {
1305
+ "entropy": 0.5074908941984176,
1306
+ "epoch": 2.545273631840796,
1307
+ "grad_norm": 2.124582290649414,
1308
+ "learning_rate": 0.00017805413932791875,
1309
+ "loss": 0.5137399673461914,
1310
+ "mean_token_accuracy": 0.8484948351979256,
1311
+ "num_tokens": 10513192.0,
1312
+ "step": 1280
1313
+ },
1314
+ {
1315
+ "entropy": 0.5202818088233471,
1316
+ "epoch": 2.5651741293532337,
1317
+ "grad_norm": 1.7741371393203735,
1318
+ "learning_rate": 0.00017764144158169856,
1319
+ "loss": 0.5218567848205566,
1320
+ "mean_token_accuracy": 0.8525593280792236,
1321
+ "num_tokens": 10597049.0,
1322
+ "step": 1290
1323
+ },
1324
+ {
1325
+ "entropy": 0.5169114828109741,
1326
+ "epoch": 2.585074626865672,
1327
+ "grad_norm": 2.03674054145813,
1328
+ "learning_rate": 0.00017722538723632773,
1329
+ "loss": 0.5342438697814942,
1330
+ "mean_token_accuracy": 0.8379098229110241,
1331
+ "num_tokens": 10679839.0,
1332
+ "step": 1300
1333
+ },
1334
+ {
1335
+ "entropy": 0.527920619212091,
1336
+ "epoch": 2.604975124378109,
1337
+ "grad_norm": 1.7481690645217896,
1338
+ "learning_rate": 0.000176805994278691,
1339
+ "loss": 0.5268249988555909,
1340
+ "mean_token_accuracy": 0.8468565516173839,
1341
+ "num_tokens": 10763374.0,
1342
+ "step": 1310
1343
+ },
1344
+ {
1345
+ "entropy": 0.5612680882215499,
1346
+ "epoch": 2.6248756218905474,
1347
+ "grad_norm": 2.044337511062622,
1348
+ "learning_rate": 0.0001763832808400082,
1349
+ "loss": 0.5637305736541748,
1350
+ "mean_token_accuracy": 0.8389740340411663,
1351
+ "num_tokens": 10846467.0,
1352
+ "step": 1320
1353
+ },
1354
+ {
1355
+ "entropy": 0.526572679169476,
1356
+ "epoch": 2.644776119402985,
1357
+ "grad_norm": 1.5558581352233887,
1358
+ "learning_rate": 0.00017595726519505043,
1359
+ "loss": 0.5200609683990478,
1360
+ "mean_token_accuracy": 0.848398495465517,
1361
+ "num_tokens": 10929729.0,
1362
+ "step": 1330
1363
+ },
1364
+ {
1365
+ "entropy": 0.5076324449852109,
1366
+ "epoch": 2.664676616915423,
1367
+ "grad_norm": 2.233233690261841,
1368
+ "learning_rate": 0.00017552796576134985,
1369
+ "loss": 0.5190268993377686,
1370
+ "mean_token_accuracy": 0.841417095810175,
1371
+ "num_tokens": 11012769.0,
1372
+ "step": 1340
1373
+ },
1374
+ {
1375
+ "entropy": 0.5502721823751926,
1376
+ "epoch": 2.684577114427861,
1377
+ "grad_norm": 1.8738890886306763,
1378
+ "learning_rate": 0.00017509540109840365,
1379
+ "loss": 0.547359848022461,
1380
+ "mean_token_accuracy": 0.8380319178104401,
1381
+ "num_tokens": 11094716.0,
1382
+ "step": 1350
1383
+ },
1384
+ {
1385
+ "entropy": 0.5275072449818253,
1386
+ "epoch": 2.7044776119402982,
1387
+ "grad_norm": 1.8149367570877075,
1388
+ "learning_rate": 0.00017465958990687156,
1389
+ "loss": 0.5218305110931396,
1390
+ "mean_token_accuracy": 0.8568139627575875,
1391
+ "num_tokens": 11175013.0,
1392
+ "step": 1360
1393
+ },
1394
+ {
1395
+ "entropy": 0.5119684131816029,
1396
+ "epoch": 2.7243781094527364,
1397
+ "grad_norm": 1.698498010635376,
1398
+ "learning_rate": 0.0001742205510277674,
1399
+ "loss": 0.5018157482147216,
1400
+ "mean_token_accuracy": 0.8495248600840568,
1401
+ "num_tokens": 11256288.0,
1402
+ "step": 1370
1403
+ },
1404
+ {
1405
+ "entropy": 0.5109101135283709,
1406
+ "epoch": 2.744278606965174,
1407
+ "grad_norm": 1.837099552154541,
1408
+ "learning_rate": 0.00017377830344164464,
1409
+ "loss": 0.5008152484893799,
1410
+ "mean_token_accuracy": 0.8569738574326038,
1411
+ "num_tokens": 11338498.0,
1412
+ "step": 1380
1413
+ },
1414
+ {
1415
+ "entropy": 0.5329740956425667,
1416
+ "epoch": 2.764179104477612,
1417
+ "grad_norm": 2.1258440017700195,
1418
+ "learning_rate": 0.00017333286626777564,
1419
+ "loss": 0.5308613777160645,
1420
+ "mean_token_accuracy": 0.8408495858311653,
1421
+ "num_tokens": 11422122.0,
1422
+ "step": 1390
1423
+ },
1424
+ {
1425
+ "entropy": 0.5458354912698269,
1426
+ "epoch": 2.78407960199005,
1427
+ "grad_norm": 2.0416526794433594,
1428
+ "learning_rate": 0.00017288425876332527,
1429
+ "loss": 0.5461023807525635,
1430
+ "mean_token_accuracy": 0.8418126352131367,
1431
+ "num_tokens": 11506227.0,
1432
+ "step": 1400
1433
+ },
1434
+ {
1435
+ "entropy": 0.5177356921136379,
1436
+ "epoch": 2.8039800995024877,
1437
+ "grad_norm": 1.8857133388519287,
1438
+ "learning_rate": 0.00017243250032251823,
1439
+ "loss": 0.5320337772369385,
1440
+ "mean_token_accuracy": 0.8437633819878101,
1441
+ "num_tokens": 11586611.0,
1442
+ "step": 1410
1443
+ },
1444
+ {
1445
+ "entropy": 0.5068972071632742,
1446
+ "epoch": 2.8238805970149254,
1447
+ "grad_norm": 2.1604552268981934,
1448
+ "learning_rate": 0.00017197761047580082,
1449
+ "loss": 0.4947951793670654,
1450
+ "mean_token_accuracy": 0.8533644363284111,
1451
+ "num_tokens": 11670179.0,
1452
+ "step": 1420
1453
+ },
1454
+ {
1455
+ "entropy": 0.491135448589921,
1456
+ "epoch": 2.843781094527363,
1457
+ "grad_norm": 1.9001272916793823,
1458
+ "learning_rate": 0.00017151960888899627,
1459
+ "loss": 0.5062966346740723,
1460
+ "mean_token_accuracy": 0.8556574188172817,
1461
+ "num_tokens": 11750106.0,
1462
+ "step": 1430
1463
+ },
1464
+ {
1465
+ "entropy": 0.5211849508807063,
1466
+ "epoch": 2.863681592039801,
1467
+ "grad_norm": 1.9398213624954224,
1468
+ "learning_rate": 0.00017105851536245492,
1469
+ "loss": 0.48212461471557616,
1470
+ "mean_token_accuracy": 0.8569056294858456,
1471
+ "num_tokens": 11831237.0,
1472
+ "step": 1440
1473
+ },
1474
+ {
1475
+ "entropy": 0.4800686825066805,
1476
+ "epoch": 2.883582089552239,
1477
+ "grad_norm": 1.8992606401443481,
1478
+ "learning_rate": 0.0001705943498301979,
1479
+ "loss": 0.5234591960906982,
1480
+ "mean_token_accuracy": 0.8473109625279903,
1481
+ "num_tokens": 11915761.0,
1482
+ "step": 1450
1483
+ },
1484
+ {
1485
+ "entropy": 0.5110673246905207,
1486
+ "epoch": 2.9034825870646768,
1487
+ "grad_norm": 1.7360588312149048,
1488
+ "learning_rate": 0.00017012713235905547,
1489
+ "loss": 0.5000662803649902,
1490
+ "mean_token_accuracy": 0.8583845689892768,
1491
+ "num_tokens": 11996666.0,
1492
+ "step": 1460
1493
+ },
1494
+ {
1495
+ "entropy": 0.49351408518850803,
1496
+ "epoch": 2.9233830845771145,
1497
+ "grad_norm": 2.0405280590057373,
1498
+ "learning_rate": 0.00016965688314779956,
1499
+ "loss": 0.5083964824676513,
1500
+ "mean_token_accuracy": 0.8502242051064968,
1501
+ "num_tokens": 12082260.0,
1502
+ "step": 1470
1503
+ },
1504
+ {
1505
+ "entropy": 0.5278301501646638,
1506
+ "epoch": 2.943283582089552,
1507
+ "grad_norm": 1.9192203283309937,
1508
+ "learning_rate": 0.00016918362252627036,
1509
+ "loss": 0.5238072872161865,
1510
+ "mean_token_accuracy": 0.8439404472708703,
1511
+ "num_tokens": 12160973.0,
1512
+ "step": 1480
1513
+ },
1514
+ {
1515
+ "entropy": 0.5115002764388918,
1516
+ "epoch": 2.96318407960199,
1517
+ "grad_norm": 1.6478546857833862,
1518
+ "learning_rate": 0.00016870737095449754,
1519
+ "loss": 0.5171586036682129,
1520
+ "mean_token_accuracy": 0.8549239777028561,
1521
+ "num_tokens": 12244781.0,
1522
+ "step": 1490
1523
+ },
1524
+ {
1525
+ "entropy": 0.4658357990905643,
1526
+ "epoch": 2.983084577114428,
1527
+ "grad_norm": 2.152078866958618,
1528
+ "learning_rate": 0.00016822814902181583,
1529
+ "loss": 0.4724470615386963,
1530
+ "mean_token_accuracy": 0.8588747374713421,
1531
+ "num_tokens": 12328430.0,
1532
+ "step": 1500
1533
+ },
1534
+ {
1535
+ "epoch": 3.0,
1536
+ "eval_entropy": 0.6918873670421052,
1537
+ "eval_loss": 1.013381004333496,
1538
+ "eval_mean_token_accuracy": 0.7516806636537824,
1539
+ "eval_num_tokens": 12397968.0,
1540
+ "eval_runtime": 89.0725,
1541
+ "eval_samples_per_second": 11.62,
1542
+ "eval_steps_per_second": 5.815,
1543
+ "step": 1509
1544
+ }
1545
+ ],
1546
+ "logging_steps": 10,
1547
+ "max_steps": 5030,
1548
+ "num_input_tokens_seen": 0,
1549
+ "num_train_epochs": 10,
1550
+ "save_steps": 500,
1551
+ "stateful_callbacks": {
1552
+ "EarlyStoppingCallback": {
1553
+ "args": {
1554
+ "early_stopping_patience": 3,
1555
+ "early_stopping_threshold": 0.0
1556
+ },
1557
+ "attributes": {
1558
+ "early_stopping_patience_counter": 0
1559
+ }
1560
+ },
1561
+ "TrainerControl": {
1562
+ "args": {
1563
+ "should_epoch_stop": false,
1564
+ "should_evaluate": false,
1565
+ "should_log": false,
1566
+ "should_save": true,
1567
+ "should_training_stop": false
1568
+ },
1569
+ "attributes": {}
1570
+ }
1571
+ },
1572
+ "total_flos": 2.6428765983473664e+16,
1573
+ "train_batch_size": 2,
1574
+ "trial_name": null,
1575
+ "trial_params": null
1576
+ }
checkpoint-1509/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a46feeb2eab16954d3fefe8cda13490c466c544ca6063b8067a5806d5b663b51
3
+ size 5841
run_metadata.json ADDED
@@ -0,0 +1,3127 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "task": "nosql2doc",
3
+ "model_name": "Salesforce/codegen-350M-multi",
4
+ "device": "cuda:0",
5
+ "checkpoint_run": "v4",
6
+ "adapter_path": "/kaggle/working/models/checkpoints/v4/nosql2doc",
7
+ "train_rows": 8040,
8
+ "eval_rows": 1035,
9
+ "train_loss": 0.5501504028135771,
10
+ "eval_loss": 1.013381004333496,
11
+ "best_eval_loss": 1.013381004333496,
12
+ "train_runtime_seconds": 13547.859,
13
+ "mlflow_run_id": null,
14
+ "filter_stats": {
15
+ "input_rows": 8040,
16
+ "kept_rows": 8040,
17
+ "skipped_rows": 0,
18
+ "skip_reasons": {}
19
+ },
20
+ "token_stats": {
21
+ "rows": 8040,
22
+ "avg_prompt_tokens": 485.68,
23
+ "avg_target_tokens": 27.33,
24
+ "avg_total_tokens": 513.01,
25
+ "max_total_tokens": 1439,
26
+ "skipped_too_long_target": 0,
27
+ "skipped_too_long_prompt": 0
28
+ },
29
+ "log_history": [
30
+ {
31
+ "loss": 3.3999290466308594,
32
+ "grad_norm": 3.4684948921203613,
33
+ "learning_rate": 7.142857142857143e-06,
34
+ "entropy": 2.8841454654932024,
35
+ "num_tokens": 84042.0,
36
+ "mean_token_accuracy": 0.36815835647284983,
37
+ "epoch": 0.01990049751243781,
38
+ "step": 10
39
+ },
40
+ {
41
+ "loss": 3.1389333724975588,
42
+ "grad_norm": 3.199768304824829,
43
+ "learning_rate": 1.5079365079365079e-05,
44
+ "entropy": 2.871115285158157,
45
+ "num_tokens": 166453.0,
46
+ "mean_token_accuracy": 0.41857333555817605,
47
+ "epoch": 0.03980099502487562,
48
+ "step": 20
49
+ },
50
+ {
51
+ "loss": 2.5403181076049806,
52
+ "grad_norm": 2.241548776626587,
53
+ "learning_rate": 2.3015873015873015e-05,
54
+ "entropy": 2.689709846675396,
55
+ "num_tokens": 248357.0,
56
+ "mean_token_accuracy": 0.45860434919595716,
57
+ "epoch": 0.05970149253731343,
58
+ "step": 30
59
+ },
60
+ {
61
+ "loss": 2.2156551361083983,
62
+ "grad_norm": 2.2443745136260986,
63
+ "learning_rate": 3.095238095238095e-05,
64
+ "entropy": 2.371776354312897,
65
+ "num_tokens": 329302.0,
66
+ "mean_token_accuracy": 0.534111600741744,
67
+ "epoch": 0.07960199004975124,
68
+ "step": 40
69
+ },
70
+ {
71
+ "loss": 1.9395181655883789,
72
+ "grad_norm": 2.8479607105255127,
73
+ "learning_rate": 3.888888888888889e-05,
74
+ "entropy": 1.9646751508116722,
75
+ "num_tokens": 410388.0,
76
+ "mean_token_accuracy": 0.5719782687723637,
77
+ "epoch": 0.09950248756218906,
78
+ "step": 50
79
+ },
80
+ {
81
+ "loss": 1.7669334411621094,
82
+ "grad_norm": 2.5392374992370605,
83
+ "learning_rate": 4.682539682539683e-05,
84
+ "entropy": 1.712027932703495,
85
+ "num_tokens": 491142.0,
86
+ "mean_token_accuracy": 0.5962413489818573,
87
+ "epoch": 0.11940298507462686,
88
+ "step": 60
89
+ },
90
+ {
91
+ "loss": 1.6198507308959962,
92
+ "grad_norm": 2.8646080493927,
93
+ "learning_rate": 5.4761904761904766e-05,
94
+ "entropy": 1.6189243733882903,
95
+ "num_tokens": 573054.0,
96
+ "mean_token_accuracy": 0.6142528053373099,
97
+ "epoch": 0.13930348258706468,
98
+ "step": 70
99
+ },
100
+ {
101
+ "loss": 1.6246612548828125,
102
+ "grad_norm": 3.0876874923706055,
103
+ "learning_rate": 6.26984126984127e-05,
104
+ "entropy": 1.6242131367325783,
105
+ "num_tokens": 658316.0,
106
+ "mean_token_accuracy": 0.626776534691453,
107
+ "epoch": 0.15920398009950248,
108
+ "step": 80
109
+ },
110
+ {
111
+ "loss": 1.5039705276489257,
112
+ "grad_norm": 2.827972650527954,
113
+ "learning_rate": 7.063492063492065e-05,
114
+ "entropy": 1.5123998656868936,
115
+ "num_tokens": 738196.0,
116
+ "mean_token_accuracy": 0.6451249852776527,
117
+ "epoch": 0.1791044776119403,
118
+ "step": 90
119
+ },
120
+ {
121
+ "loss": 1.4707328796386718,
122
+ "grad_norm": 2.794956684112549,
123
+ "learning_rate": 7.857142857142858e-05,
124
+ "entropy": 1.4389337845146657,
125
+ "num_tokens": 818366.0,
126
+ "mean_token_accuracy": 0.6486270293593407,
127
+ "epoch": 0.19900497512437812,
128
+ "step": 100
129
+ },
130
+ {
131
+ "loss": 1.461697483062744,
132
+ "grad_norm": 2.650611639022827,
133
+ "learning_rate": 8.650793650793651e-05,
134
+ "entropy": 1.4290485948324203,
135
+ "num_tokens": 900542.0,
136
+ "mean_token_accuracy": 0.6452589213848114,
137
+ "epoch": 0.21890547263681592,
138
+ "step": 110
139
+ },
140
+ {
141
+ "loss": 1.3799741744995118,
142
+ "grad_norm": 3.212292194366455,
143
+ "learning_rate": 9.444444444444444e-05,
144
+ "entropy": 1.4537153281271458,
145
+ "num_tokens": 983783.0,
146
+ "mean_token_accuracy": 0.665483495220542,
147
+ "epoch": 0.23880597014925373,
148
+ "step": 120
149
+ },
150
+ {
151
+ "loss": 1.339876365661621,
152
+ "grad_norm": 2.367784261703491,
153
+ "learning_rate": 0.00010238095238095237,
154
+ "entropy": 1.3213311471045017,
155
+ "num_tokens": 1065282.0,
156
+ "mean_token_accuracy": 0.6738482430577278,
157
+ "epoch": 0.25870646766169153,
158
+ "step": 130
159
+ },
160
+ {
161
+ "loss": 1.3189430236816406,
162
+ "grad_norm": 2.7096056938171387,
163
+ "learning_rate": 0.00011031746031746033,
164
+ "entropy": 1.2877162352204323,
165
+ "num_tokens": 1147430.0,
166
+ "mean_token_accuracy": 0.6620625633746385,
167
+ "epoch": 0.27860696517412936,
168
+ "step": 140
169
+ },
170
+ {
171
+ "loss": 1.3049938201904296,
172
+ "grad_norm": 2.497452974319458,
173
+ "learning_rate": 0.00011825396825396826,
174
+ "entropy": 1.3201022632420063,
175
+ "num_tokens": 1225840.0,
176
+ "mean_token_accuracy": 0.6833859849721193,
177
+ "epoch": 0.29850746268656714,
178
+ "step": 150
179
+ },
180
+ {
181
+ "loss": 1.231965923309326,
182
+ "grad_norm": 3.0387496948242188,
183
+ "learning_rate": 0.0001261904761904762,
184
+ "entropy": 1.2007936105132102,
185
+ "num_tokens": 1309268.0,
186
+ "mean_token_accuracy": 0.7017441529780626,
187
+ "epoch": 0.31840796019900497,
188
+ "step": 160
189
+ },
190
+ {
191
+ "loss": 1.2840826034545898,
192
+ "grad_norm": 2.327012300491333,
193
+ "learning_rate": 0.00013412698412698412,
194
+ "entropy": 1.3119081839919091,
195
+ "num_tokens": 1389094.0,
196
+ "mean_token_accuracy": 0.6792996764183045,
197
+ "epoch": 0.3383084577114428,
198
+ "step": 170
199
+ },
200
+ {
201
+ "loss": 1.2992033958435059,
202
+ "grad_norm": 2.426831007003784,
203
+ "learning_rate": 0.00014206349206349208,
204
+ "entropy": 1.2995639897882938,
205
+ "num_tokens": 1471243.0,
206
+ "mean_token_accuracy": 0.6910909987986088,
207
+ "epoch": 0.3582089552238806,
208
+ "step": 180
209
+ },
210
+ {
211
+ "loss": 1.1649029731750489,
212
+ "grad_norm": 2.783355951309204,
213
+ "learning_rate": 0.00015000000000000001,
214
+ "entropy": 1.1952705256640912,
215
+ "num_tokens": 1552783.0,
216
+ "mean_token_accuracy": 0.7020247872918844,
217
+ "epoch": 0.3781094527363184,
218
+ "step": 190
219
+ },
220
+ {
221
+ "loss": 1.2627052307128905,
222
+ "grad_norm": 2.162163019180298,
223
+ "learning_rate": 0.00015793650793650795,
224
+ "entropy": 1.2681411176919937,
225
+ "num_tokens": 1635174.0,
226
+ "mean_token_accuracy": 0.6855453528463841,
227
+ "epoch": 0.39800995024875624,
228
+ "step": 200
229
+ },
230
+ {
231
+ "loss": 1.237012004852295,
232
+ "grad_norm": 2.3045036792755127,
233
+ "learning_rate": 0.0001658730158730159,
234
+ "entropy": 1.2593041777610778,
235
+ "num_tokens": 1720930.0,
236
+ "mean_token_accuracy": 0.6885611288249492,
237
+ "epoch": 0.417910447761194,
238
+ "step": 210
239
+ },
240
+ {
241
+ "loss": 1.2200750350952148,
242
+ "grad_norm": 2.291395664215088,
243
+ "learning_rate": 0.00017380952380952383,
244
+ "entropy": 1.1904637023806572,
245
+ "num_tokens": 1803132.0,
246
+ "mean_token_accuracy": 0.6859784577041864,
247
+ "epoch": 0.43781094527363185,
248
+ "step": 220
249
+ },
250
+ {
251
+ "loss": 1.2164555549621583,
252
+ "grad_norm": 2.553251266479492,
253
+ "learning_rate": 0.00018174603174603177,
254
+ "entropy": 1.1843627236783505,
255
+ "num_tokens": 1884825.0,
256
+ "mean_token_accuracy": 0.6943051021546125,
257
+ "epoch": 0.4577114427860697,
258
+ "step": 230
259
+ },
260
+ {
261
+ "loss": 1.1573832511901856,
262
+ "grad_norm": 2.403580904006958,
263
+ "learning_rate": 0.0001896825396825397,
264
+ "entropy": 1.1876815035939217,
265
+ "num_tokens": 1968914.0,
266
+ "mean_token_accuracy": 0.7180207531899214,
267
+ "epoch": 0.47761194029850745,
268
+ "step": 240
269
+ },
270
+ {
271
+ "loss": 1.1767531394958497,
272
+ "grad_norm": 2.3557302951812744,
273
+ "learning_rate": 0.00019761904761904763,
274
+ "entropy": 1.1310456305742265,
275
+ "num_tokens": 2050602.0,
276
+ "mean_token_accuracy": 0.6986724361777306,
277
+ "epoch": 0.4975124378109453,
278
+ "step": 250
279
+ },
280
+ {
281
+ "loss": 1.187550926208496,
282
+ "grad_norm": 2.430337905883789,
283
+ "learning_rate": 0.0001999989408126818,
284
+ "entropy": 1.2115244895219803,
285
+ "num_tokens": 2134774.0,
286
+ "mean_token_accuracy": 0.6943733543157578,
287
+ "epoch": 0.5174129353233831,
288
+ "step": 260
289
+ },
290
+ {
291
+ "loss": 1.1683053970336914,
292
+ "grad_norm": 1.848976969718933,
293
+ "learning_rate": 0.0001999937530104439,
294
+ "entropy": 1.1603886000812054,
295
+ "num_tokens": 2214022.0,
296
+ "mean_token_accuracy": 0.6958795577287674,
297
+ "epoch": 0.5373134328358209,
298
+ "step": 270
299
+ },
300
+ {
301
+ "loss": 1.087782859802246,
302
+ "grad_norm": 2.297624111175537,
303
+ "learning_rate": 0.00019998424227267588,
304
+ "entropy": 1.083036269247532,
305
+ "num_tokens": 2295817.0,
306
+ "mean_token_accuracy": 0.7164598941802979,
307
+ "epoch": 0.5572139303482587,
308
+ "step": 280
309
+ },
310
+ {
311
+ "loss": 1.1336830139160157,
312
+ "grad_norm": 2.1926214694976807,
313
+ "learning_rate": 0.00019997040901054646,
314
+ "entropy": 1.1467085666954517,
315
+ "num_tokens": 2379081.0,
316
+ "mean_token_accuracy": 0.7070459358394146,
317
+ "epoch": 0.5771144278606966,
318
+ "step": 290
319
+ },
320
+ {
321
+ "loss": 1.104503059387207,
322
+ "grad_norm": 2.108574390411377,
323
+ "learning_rate": 0.000199952253822096,
324
+ "entropy": 1.1092145919799805,
325
+ "num_tokens": 2461290.0,
326
+ "mean_token_accuracy": 0.7167623668909073,
327
+ "epoch": 0.5970149253731343,
328
+ "step": 300
329
+ },
330
+ {
331
+ "loss": 1.0758417129516602,
332
+ "grad_norm": 1.9235388040542603,
333
+ "learning_rate": 0.00019992977749221064,
334
+ "entropy": 1.0762298285961152,
335
+ "num_tokens": 2546927.0,
336
+ "mean_token_accuracy": 0.7244217373430729,
337
+ "epoch": 0.6169154228855721,
338
+ "step": 310
339
+ },
340
+ {
341
+ "loss": 1.0714119911193847,
342
+ "grad_norm": 2.015493154525757,
343
+ "learning_rate": 0.0001999029809925883,
344
+ "entropy": 1.058315774053335,
345
+ "num_tokens": 2627151.0,
346
+ "mean_token_accuracy": 0.7185824103653431,
347
+ "epoch": 0.6368159203980099,
348
+ "step": 320
349
+ },
350
+ {
351
+ "loss": 1.0344210624694825,
352
+ "grad_norm": 1.824608564376831,
353
+ "learning_rate": 0.00019987186548169682,
354
+ "entropy": 1.040397210419178,
355
+ "num_tokens": 2711367.0,
356
+ "mean_token_accuracy": 0.7333592250943184,
357
+ "epoch": 0.6567164179104478,
358
+ "step": 330
359
+ },
360
+ {
361
+ "loss": 0.9733485221862793,
362
+ "grad_norm": 1.9432896375656128,
363
+ "learning_rate": 0.0001998364323047236,
364
+ "entropy": 0.9891116008162498,
365
+ "num_tokens": 2793468.0,
366
+ "mean_token_accuracy": 0.7416356436908245,
367
+ "epoch": 0.6766169154228856,
368
+ "step": 340
369
+ },
370
+ {
371
+ "loss": 1.0494978904724122,
372
+ "grad_norm": 1.906457781791687,
373
+ "learning_rate": 0.00019979668299351783,
374
+ "entropy": 1.0004291333258153,
375
+ "num_tokens": 2875166.0,
376
+ "mean_token_accuracy": 0.7176768533885479,
377
+ "epoch": 0.6965174129353234,
378
+ "step": 350
379
+ },
380
+ {
381
+ "loss": 1.0371734619140625,
382
+ "grad_norm": 1.7709777355194092,
383
+ "learning_rate": 0.00019975261926652392,
384
+ "entropy": 1.0571318455040455,
385
+ "num_tokens": 2959671.0,
386
+ "mean_token_accuracy": 0.7288566574454307,
387
+ "epoch": 0.7164179104477612,
388
+ "step": 360
389
+ },
390
+ {
391
+ "loss": 0.9970813751220703,
392
+ "grad_norm": 1.896713376045227,
393
+ "learning_rate": 0.00019970424302870739,
394
+ "entropy": 0.9674378834664822,
395
+ "num_tokens": 3041437.0,
396
+ "mean_token_accuracy": 0.7395003981888294,
397
+ "epoch": 0.736318407960199,
398
+ "step": 370
399
+ },
400
+ {
401
+ "loss": 1.0752775192260742,
402
+ "grad_norm": 1.7242449522018433,
403
+ "learning_rate": 0.00019965155637147243,
404
+ "entropy": 1.1270258143544196,
405
+ "num_tokens": 3123865.0,
406
+ "mean_token_accuracy": 0.7305668473243714,
407
+ "epoch": 0.7562189054726368,
408
+ "step": 380
409
+ },
410
+ {
411
+ "loss": 0.9751872062683106,
412
+ "grad_norm": 1.781772255897522,
413
+ "learning_rate": 0.0001995945615725716,
414
+ "entropy": 0.9315242812037468,
415
+ "num_tokens": 3205032.0,
416
+ "mean_token_accuracy": 0.7497406356036663,
417
+ "epoch": 0.7761194029850746,
418
+ "step": 390
419
+ },
420
+ {
421
+ "loss": 1.0796968460083007,
422
+ "grad_norm": 1.8422024250030518,
423
+ "learning_rate": 0.00019953326109600728,
424
+ "entropy": 1.076903734356165,
425
+ "num_tokens": 3286003.0,
426
+ "mean_token_accuracy": 0.7176821112632752,
427
+ "epoch": 0.7960199004975125,
428
+ "step": 400
429
+ },
430
+ {
431
+ "loss": 0.9697890281677246,
432
+ "grad_norm": 1.7082111835479736,
433
+ "learning_rate": 0.00019946765759192497,
434
+ "entropy": 0.9634652934968472,
435
+ "num_tokens": 3367484.0,
436
+ "mean_token_accuracy": 0.7406083434820175,
437
+ "epoch": 0.8159203980099502,
438
+ "step": 410
439
+ },
440
+ {
441
+ "loss": 0.9277518272399903,
442
+ "grad_norm": 1.8066761493682861,
443
+ "learning_rate": 0.00019939775389649916,
444
+ "entropy": 0.9321402795612812,
445
+ "num_tokens": 3453904.0,
446
+ "mean_token_accuracy": 0.7574292354285717,
447
+ "epoch": 0.835820895522388,
448
+ "step": 420
449
+ },
450
+ {
451
+ "loss": 0.9705208778381348,
452
+ "grad_norm": 1.6425416469573975,
453
+ "learning_rate": 0.00019932355303181026,
454
+ "entropy": 0.9754443395882845,
455
+ "num_tokens": 3536472.0,
456
+ "mean_token_accuracy": 0.7435099229216575,
457
+ "epoch": 0.8557213930348259,
458
+ "step": 430
459
+ },
460
+ {
461
+ "loss": 1.036739444732666,
462
+ "grad_norm": 1.707189679145813,
463
+ "learning_rate": 0.00019924505820571425,
464
+ "entropy": 0.9742012087255716,
465
+ "num_tokens": 3616711.0,
466
+ "mean_token_accuracy": 0.7312861289829016,
467
+ "epoch": 0.8756218905472637,
468
+ "step": 440
469
+ },
470
+ {
471
+ "loss": 1.0217846870422362,
472
+ "grad_norm": 1.865692377090454,
473
+ "learning_rate": 0.0001991622728117038,
474
+ "entropy": 1.0537842728197575,
475
+ "num_tokens": 3701126.0,
476
+ "mean_token_accuracy": 0.7338800489902496,
477
+ "epoch": 0.8955223880597015,
478
+ "step": 450
479
+ },
480
+ {
481
+ "loss": 1.0003718376159667,
482
+ "grad_norm": 1.7353651523590088,
483
+ "learning_rate": 0.00019907520042876172,
484
+ "entropy": 0.989534855633974,
485
+ "num_tokens": 3783495.0,
486
+ "mean_token_accuracy": 0.7480997510254384,
487
+ "epoch": 0.9154228855721394,
488
+ "step": 460
489
+ },
490
+ {
491
+ "loss": 1.0100757598876953,
492
+ "grad_norm": 2.007262706756592,
493
+ "learning_rate": 0.00019898384482120614,
494
+ "entropy": 0.9294702004641294,
495
+ "num_tokens": 3863784.0,
496
+ "mean_token_accuracy": 0.7364842720329762,
497
+ "epoch": 0.9353233830845771,
498
+ "step": 470
499
+ },
500
+ {
501
+ "loss": 0.9432971000671386,
502
+ "grad_norm": 1.9396175146102905,
503
+ "learning_rate": 0.0001988882099385278,
504
+ "entropy": 0.9523782070726157,
505
+ "num_tokens": 3947540.0,
506
+ "mean_token_accuracy": 0.756897260248661,
507
+ "epoch": 0.9552238805970149,
508
+ "step": 480
509
+ },
510
+ {
511
+ "loss": 0.9125359535217286,
512
+ "grad_norm": 1.8582719564437866,
513
+ "learning_rate": 0.00019878829991521935,
514
+ "entropy": 0.9421051684767008,
515
+ "num_tokens": 4031284.0,
516
+ "mean_token_accuracy": 0.7575220972299576,
517
+ "epoch": 0.9751243781094527,
518
+ "step": 490
519
+ },
520
+ {
521
+ "loss": 0.9656861305236817,
522
+ "grad_norm": 1.9623899459838867,
523
+ "learning_rate": 0.00019868411907059645,
524
+ "entropy": 0.9381607126444578,
525
+ "num_tokens": 4112149.0,
526
+ "mean_token_accuracy": 0.7544871643185616,
527
+ "epoch": 0.9950248756218906,
528
+ "step": 500
529
+ },
530
+ {
531
+ "eval_loss": 1.063795566558838,
532
+ "eval_runtime": 88.6229,
533
+ "eval_samples_per_second": 11.679,
534
+ "eval_steps_per_second": 5.845,
535
+ "eval_entropy": 0.9874700385516214,
536
+ "eval_num_tokens": 4132656.0,
537
+ "eval_mean_token_accuracy": 0.7334998920847551,
538
+ "epoch": 1.0,
539
+ "step": 503
540
+ },
541
+ {
542
+ "loss": 0.7886298179626465,
543
+ "grad_norm": 1.6549201011657715,
544
+ "learning_rate": 0.00019857567190861126,
545
+ "entropy": 0.8532990099568116,
546
+ "num_tokens": 4190723.0,
547
+ "mean_token_accuracy": 0.7860889779893976,
548
+ "epoch": 1.0139303482587065,
549
+ "step": 510
550
+ },
551
+ {
552
+ "loss": 0.777919864654541,
553
+ "grad_norm": 1.6148929595947266,
554
+ "learning_rate": 0.00019846296311765754,
555
+ "entropy": 0.7671804554760456,
556
+ "num_tokens": 4273358.0,
557
+ "mean_token_accuracy": 0.789631099998951,
558
+ "epoch": 1.0338308457711443,
559
+ "step": 520
560
+ },
561
+ {
562
+ "loss": 0.7967178821563721,
563
+ "grad_norm": 1.765950322151184,
564
+ "learning_rate": 0.00019834599757036803,
565
+ "entropy": 0.8041674952954054,
566
+ "num_tokens": 4356030.0,
567
+ "mean_token_accuracy": 0.784614659100771,
568
+ "epoch": 1.053731343283582,
569
+ "step": 530
570
+ },
571
+ {
572
+ "loss": 0.7301093578338623,
573
+ "grad_norm": 1.7868931293487549,
574
+ "learning_rate": 0.00019822478032340387,
575
+ "entropy": 0.7405303593724966,
576
+ "num_tokens": 4440157.0,
577
+ "mean_token_accuracy": 0.8020888641476631,
578
+ "epoch": 1.07363184079602,
579
+ "step": 540
580
+ },
581
+ {
582
+ "loss": 0.7884031295776367,
583
+ "grad_norm": 1.991782784461975,
584
+ "learning_rate": 0.0001980993166172358,
585
+ "entropy": 0.7550990927964449,
586
+ "num_tokens": 4523343.0,
587
+ "mean_token_accuracy": 0.7879343688488006,
588
+ "epoch": 1.0935323383084576,
589
+ "step": 550
590
+ },
591
+ {
592
+ "loss": 0.8159590721130371,
593
+ "grad_norm": 1.869396686553955,
594
+ "learning_rate": 0.00019796961187591781,
595
+ "entropy": 0.812371177971363,
596
+ "num_tokens": 4605454.0,
597
+ "mean_token_accuracy": 0.7711428456008435,
598
+ "epoch": 1.1134328358208956,
599
+ "step": 560
600
+ },
601
+ {
602
+ "loss": 0.8079433441162109,
603
+ "grad_norm": 1.9530060291290283,
604
+ "learning_rate": 0.00019783567170685262,
605
+ "entropy": 0.7879433136433363,
606
+ "num_tokens": 4687318.0,
607
+ "mean_token_accuracy": 0.7875082984566688,
608
+ "epoch": 1.1333333333333333,
609
+ "step": 570
610
+ },
611
+ {
612
+ "loss": 0.8188363075256347,
613
+ "grad_norm": 1.7895166873931885,
614
+ "learning_rate": 0.00019769750190054905,
615
+ "entropy": 0.8396137218922377,
616
+ "num_tokens": 4768232.0,
617
+ "mean_token_accuracy": 0.7710263684391976,
618
+ "epoch": 1.153233830845771,
619
+ "step": 580
620
+ },
621
+ {
622
+ "loss": 0.8532954216003418,
623
+ "grad_norm": 1.8147255182266235,
624
+ "learning_rate": 0.00019755510843037191,
625
+ "entropy": 0.8406378872692585,
626
+ "num_tokens": 4850366.0,
627
+ "mean_token_accuracy": 0.7680545568466186,
628
+ "epoch": 1.173134328358209,
629
+ "step": 590
630
+ },
631
+ {
632
+ "loss": 0.7972042560577393,
633
+ "grad_norm": 1.9839149713516235,
634
+ "learning_rate": 0.00019740849745228367,
635
+ "entropy": 0.7758986987173557,
636
+ "num_tokens": 4932940.0,
637
+ "mean_token_accuracy": 0.77668926268816,
638
+ "epoch": 1.1930348258706467,
639
+ "step": 600
640
+ },
641
+ {
642
+ "loss": 0.7664913654327392,
643
+ "grad_norm": 1.776328444480896,
644
+ "learning_rate": 0.00019725767530457837,
645
+ "entropy": 0.825455692410469,
646
+ "num_tokens": 5012821.0,
647
+ "mean_token_accuracy": 0.7940710753202438,
648
+ "epoch": 1.2129353233830846,
649
+ "step": 610
650
+ },
651
+ {
652
+ "loss": 0.7731894969940185,
653
+ "grad_norm": 2.0241360664367676,
654
+ "learning_rate": 0.00019710264850760756,
655
+ "entropy": 0.7286036755889654,
656
+ "num_tokens": 5095340.0,
657
+ "mean_token_accuracy": 0.7846131704747676,
658
+ "epoch": 1.2328358208955223,
659
+ "step": 620
660
+ },
661
+ {
662
+ "loss": 0.7303972244262695,
663
+ "grad_norm": 1.7310129404067993,
664
+ "learning_rate": 0.00019694342376349835,
665
+ "entropy": 0.7920811083167791,
666
+ "num_tokens": 5177547.0,
667
+ "mean_token_accuracy": 0.7983584970235824,
668
+ "epoch": 1.25273631840796,
669
+ "step": 630
670
+ },
671
+ {
672
+ "loss": 0.8018975257873535,
673
+ "grad_norm": 1.7608613967895508,
674
+ "learning_rate": 0.00019678000795586386,
675
+ "entropy": 0.7178942065685987,
676
+ "num_tokens": 5265612.0,
677
+ "mean_token_accuracy": 0.7767767131328582,
678
+ "epoch": 1.272636815920398,
679
+ "step": 640
680
+ },
681
+ {
682
+ "loss": 0.7223423480987549,
683
+ "grad_norm": 2.0576155185699463,
684
+ "learning_rate": 0.00019661240814950536,
685
+ "entropy": 0.8001765824854374,
686
+ "num_tokens": 5348652.0,
687
+ "mean_token_accuracy": 0.7918425254523754,
688
+ "epoch": 1.292537313432836,
689
+ "step": 650
690
+ },
691
+ {
692
+ "loss": 0.8217278480529785,
693
+ "grad_norm": 1.539169430732727,
694
+ "learning_rate": 0.00019644063159010716,
695
+ "entropy": 0.7580439992249012,
696
+ "num_tokens": 5431834.0,
697
+ "mean_token_accuracy": 0.7793662197887897,
698
+ "epoch": 1.3124378109452737,
699
+ "step": 660
700
+ },
701
+ {
702
+ "loss": 0.7759135723114013,
703
+ "grad_norm": 1.8933053016662598,
704
+ "learning_rate": 0.00019626468570392298,
705
+ "entropy": 0.8075141672044992,
706
+ "num_tokens": 5514228.0,
707
+ "mean_token_accuracy": 0.7861333280801773,
708
+ "epoch": 1.3323383084577114,
709
+ "step": 670
710
+ },
711
+ {
712
+ "loss": 0.8088616371154785,
713
+ "grad_norm": 1.7051018476486206,
714
+ "learning_rate": 0.00019608457809745537,
715
+ "entropy": 0.7623774103820324,
716
+ "num_tokens": 5594702.0,
717
+ "mean_token_accuracy": 0.7746041730046272,
718
+ "epoch": 1.3522388059701491,
719
+ "step": 680
720
+ },
721
+ {
722
+ "loss": 0.7946909904479981,
723
+ "grad_norm": 1.8061248064041138,
724
+ "learning_rate": 0.00019590031655712631,
725
+ "entropy": 0.8089807592332363,
726
+ "num_tokens": 5676309.0,
727
+ "mean_token_accuracy": 0.7781320951879025,
728
+ "epoch": 1.372139303482587,
729
+ "step": 690
730
+ },
731
+ {
732
+ "loss": 0.7615675926208496,
733
+ "grad_norm": 1.9005513191223145,
734
+ "learning_rate": 0.00019571190904894115,
735
+ "entropy": 0.7627177253365517,
736
+ "num_tokens": 5757314.0,
737
+ "mean_token_accuracy": 0.788880155980587,
738
+ "epoch": 1.392039800995025,
739
+ "step": 700
740
+ },
741
+ {
742
+ "loss": 0.7611291408538818,
743
+ "grad_norm": 1.9883071184158325,
744
+ "learning_rate": 0.00019551936371814375,
745
+ "entropy": 0.7580569207668304,
746
+ "num_tokens": 5838877.0,
747
+ "mean_token_accuracy": 0.7861187107861042,
748
+ "epoch": 1.4119402985074627,
749
+ "step": 710
750
+ },
751
+ {
752
+ "loss": 0.7204749584197998,
753
+ "grad_norm": 2.187103033065796,
754
+ "learning_rate": 0.0001953226888888647,
755
+ "entropy": 0.7182941000908614,
756
+ "num_tokens": 5920820.0,
757
+ "mean_token_accuracy": 0.7946790263056756,
758
+ "epoch": 1.4318407960199004,
759
+ "step": 720
760
+ },
761
+ {
762
+ "loss": 0.757196044921875,
763
+ "grad_norm": 1.7006845474243164,
764
+ "learning_rate": 0.00019512189306376118,
765
+ "entropy": 0.6957223568111658,
766
+ "num_tokens": 6004382.0,
767
+ "mean_token_accuracy": 0.7921811021864414,
768
+ "epoch": 1.4517412935323384,
769
+ "step": 730
770
+ },
771
+ {
772
+ "loss": 0.7292089462280273,
773
+ "grad_norm": 1.6335111856460571,
774
+ "learning_rate": 0.0001949169849236496,
775
+ "entropy": 0.7850441809743642,
776
+ "num_tokens": 6086449.0,
777
+ "mean_token_accuracy": 0.797366950660944,
778
+ "epoch": 1.471641791044776,
779
+ "step": 740
780
+ },
781
+ {
782
+ "loss": 0.8179457664489747,
783
+ "grad_norm": 1.7224810123443604,
784
+ "learning_rate": 0.00019470797332713012,
785
+ "entropy": 0.7639980979263783,
786
+ "num_tokens": 6168749.0,
787
+ "mean_token_accuracy": 0.7910384394228458,
788
+ "epoch": 1.491542288557214,
789
+ "step": 750
790
+ },
791
+ {
792
+ "loss": 0.7916707515716552,
793
+ "grad_norm": 1.9208451509475708,
794
+ "learning_rate": 0.0001944948673102038,
795
+ "entropy": 0.8024922411888837,
796
+ "num_tokens": 6248672.0,
797
+ "mean_token_accuracy": 0.7822470977902413,
798
+ "epoch": 1.5114427860696518,
799
+ "step": 760
800
+ },
801
+ {
802
+ "loss": 0.7538708209991455,
803
+ "grad_norm": 1.9618816375732422,
804
+ "learning_rate": 0.00019427767608588183,
805
+ "entropy": 0.7430106606334448,
806
+ "num_tokens": 6331261.0,
807
+ "mean_token_accuracy": 0.7816703900694847,
808
+ "epoch": 1.5313432835820895,
809
+ "step": 770
810
+ },
811
+ {
812
+ "loss": 0.7316685676574707,
813
+ "grad_norm": 1.6858505010604858,
814
+ "learning_rate": 0.0001940564090437875,
815
+ "entropy": 0.7457086000591516,
816
+ "num_tokens": 6415504.0,
817
+ "mean_token_accuracy": 0.8001063913106918,
818
+ "epoch": 1.5512437810945272,
819
+ "step": 780
820
+ },
821
+ {
822
+ "loss": 0.7277198791503906,
823
+ "grad_norm": 1.9956564903259277,
824
+ "learning_rate": 0.00019383107574974984,
825
+ "entropy": 0.7096565395593644,
826
+ "num_tokens": 6497199.0,
827
+ "mean_token_accuracy": 0.7895878478884697,
828
+ "epoch": 1.5711442786069652,
829
+ "step": 790
830
+ },
831
+ {
832
+ "loss": 0.7483164310455322,
833
+ "grad_norm": 1.7207188606262207,
834
+ "learning_rate": 0.00019360168594539055,
835
+ "entropy": 0.7606754396110773,
836
+ "num_tokens": 6577494.0,
837
+ "mean_token_accuracy": 0.7934170700609684,
838
+ "epoch": 1.591044776119403,
839
+ "step": 800
840
+ },
841
+ {
842
+ "loss": 0.7517632007598877,
843
+ "grad_norm": 1.7885297536849976,
844
+ "learning_rate": 0.0001933682495477024,
845
+ "entropy": 0.7786010228097439,
846
+ "num_tokens": 6660010.0,
847
+ "mean_token_accuracy": 0.791710589081049,
848
+ "epoch": 1.6109452736318408,
849
+ "step": 810
850
+ },
851
+ {
852
+ "loss": 0.7747976303100585,
853
+ "grad_norm": 1.929076910018921,
854
+ "learning_rate": 0.00019313077664862092,
855
+ "entropy": 0.73126558996737,
856
+ "num_tokens": 6739669.0,
857
+ "mean_token_accuracy": 0.7895058654248714,
858
+ "epoch": 1.6308457711442785,
859
+ "step": 820
860
+ },
861
+ {
862
+ "loss": 0.6981019020080567,
863
+ "grad_norm": 1.6692851781845093,
864
+ "learning_rate": 0.00019288927751458766,
865
+ "entropy": 0.7332248043268919,
866
+ "num_tokens": 6819838.0,
867
+ "mean_token_accuracy": 0.8055072821676731,
868
+ "epoch": 1.6507462686567163,
869
+ "step": 830
870
+ },
871
+ {
872
+ "loss": 0.7524178504943848,
873
+ "grad_norm": 1.5395867824554443,
874
+ "learning_rate": 0.0001926437625861068,
875
+ "entropy": 0.712925647944212,
876
+ "num_tokens": 6901777.0,
877
+ "mean_token_accuracy": 0.7908283203840256,
878
+ "epoch": 1.6706467661691542,
879
+ "step": 840
880
+ },
881
+ {
882
+ "loss": 0.7306118488311768,
883
+ "grad_norm": 1.6641769409179688,
884
+ "learning_rate": 0.00019239424247729345,
885
+ "entropy": 0.7585150092840195,
886
+ "num_tokens": 6983222.0,
887
+ "mean_token_accuracy": 0.7910356394946575,
888
+ "epoch": 1.6905472636815921,
889
+ "step": 850
890
+ },
891
+ {
892
+ "loss": 0.752435302734375,
893
+ "grad_norm": 1.9537489414215088,
894
+ "learning_rate": 0.0001921407279754149,
895
+ "entropy": 0.7012663371860981,
896
+ "num_tokens": 7067291.0,
897
+ "mean_token_accuracy": 0.8054570883512497,
898
+ "epoch": 1.7104477611940299,
899
+ "step": 860
900
+ },
901
+ {
902
+ "loss": 0.6932102680206299,
903
+ "grad_norm": 1.8359386920928955,
904
+ "learning_rate": 0.00019188323004042435,
905
+ "entropy": 0.7264402851462364,
906
+ "num_tokens": 7150268.0,
907
+ "mean_token_accuracy": 0.802193396538496,
908
+ "epoch": 1.7303482587064676,
909
+ "step": 870
910
+ },
911
+ {
912
+ "loss": 0.7538199424743652,
913
+ "grad_norm": 1.9146333932876587,
914
+ "learning_rate": 0.00019162175980448688,
915
+ "entropy": 0.6838662784546614,
916
+ "num_tokens": 7231091.0,
917
+ "mean_token_accuracy": 0.796929395198822,
918
+ "epoch": 1.7502487562189055,
919
+ "step": 880
920
+ },
921
+ {
922
+ "loss": 0.7104074954986572,
923
+ "grad_norm": 2.0510952472686768,
924
+ "learning_rate": 0.0001913563285714984,
925
+ "entropy": 0.7262898899614811,
926
+ "num_tokens": 7313340.0,
927
+ "mean_token_accuracy": 0.8008449010550975,
928
+ "epoch": 1.7701492537313432,
929
+ "step": 890
930
+ },
931
+ {
932
+ "loss": 0.8105827331542969,
933
+ "grad_norm": 2.0180599689483643,
934
+ "learning_rate": 0.0001910869478165969,
935
+ "entropy": 0.7481670882552862,
936
+ "num_tokens": 7393519.0,
937
+ "mean_token_accuracy": 0.7843330048024655,
938
+ "epoch": 1.7900497512437812,
939
+ "step": 900
940
+ },
941
+ {
942
+ "loss": 0.7446431636810302,
943
+ "grad_norm": 1.646704912185669,
944
+ "learning_rate": 0.00019081362918566618,
945
+ "entropy": 0.7768452275544405,
946
+ "num_tokens": 7475016.0,
947
+ "mean_token_accuracy": 0.8008080549538136,
948
+ "epoch": 1.809950248756219,
949
+ "step": 910
950
+ },
951
+ {
952
+ "loss": 0.7266605854034424,
953
+ "grad_norm": 1.638253092765808,
954
+ "learning_rate": 0.00019053638449483259,
955
+ "entropy": 0.683229249343276,
956
+ "num_tokens": 7559243.0,
957
+ "mean_token_accuracy": 0.7936579927802085,
958
+ "epoch": 1.8298507462686566,
959
+ "step": 920
960
+ },
961
+ {
962
+ "loss": 0.738245677947998,
963
+ "grad_norm": 1.6552717685699463,
964
+ "learning_rate": 0.0001902552257299542,
965
+ "entropy": 0.7504678606987,
966
+ "num_tokens": 7643029.0,
967
+ "mean_token_accuracy": 0.7894984453916549,
968
+ "epoch": 1.8497512437810946,
969
+ "step": 930
970
+ },
971
+ {
972
+ "loss": 0.7275553226470948,
973
+ "grad_norm": 1.4619513750076294,
974
+ "learning_rate": 0.00018997016504610246,
975
+ "entropy": 0.7069665104150772,
976
+ "num_tokens": 7726479.0,
977
+ "mean_token_accuracy": 0.8092396840453148,
978
+ "epoch": 1.8696517412935323,
979
+ "step": 940
980
+ },
981
+ {
982
+ "loss": 0.6958948612213135,
983
+ "grad_norm": 1.6785398721694946,
984
+ "learning_rate": 0.00018968121476703678,
985
+ "entropy": 0.6909922767430544,
986
+ "num_tokens": 7808235.0,
987
+ "mean_token_accuracy": 0.8184511929750442,
988
+ "epoch": 1.8895522388059702,
989
+ "step": 950
990
+ },
991
+ {
992
+ "loss": 0.7307656288146973,
993
+ "grad_norm": 1.8156358003616333,
994
+ "learning_rate": 0.00018938838738467184,
995
+ "entropy": 0.7206986505538225,
996
+ "num_tokens": 7888802.0,
997
+ "mean_token_accuracy": 0.795404677093029,
998
+ "epoch": 1.909452736318408,
999
+ "step": 960
1000
+ },
1001
+ {
1002
+ "loss": 0.6951888084411622,
1003
+ "grad_norm": 1.9570693969726562,
1004
+ "learning_rate": 0.00018909169555853743,
1005
+ "entropy": 0.7154657423496247,
1006
+ "num_tokens": 7970204.0,
1007
+ "mean_token_accuracy": 0.7979383312165738,
1008
+ "epoch": 1.9293532338308457,
1009
+ "step": 970
1010
+ },
1011
+ {
1012
+ "loss": 0.6544716358184814,
1013
+ "grad_norm": 1.5071886777877808,
1014
+ "learning_rate": 0.00018879115211523117,
1015
+ "entropy": 0.6446726582944393,
1016
+ "num_tokens": 8053307.0,
1017
+ "mean_token_accuracy": 0.8045676186680794,
1018
+ "epoch": 1.9492537313432836,
1019
+ "step": 980
1020
+ },
1021
+ {
1022
+ "loss": 0.7055366516113282,
1023
+ "grad_norm": 1.4266027212142944,
1024
+ "learning_rate": 0.0001884867700478641,
1025
+ "entropy": 0.699823185056448,
1026
+ "num_tokens": 8135193.0,
1027
+ "mean_token_accuracy": 0.7992964766919612,
1028
+ "epoch": 1.9691542288557216,
1029
+ "step": 990
1030
+ },
1031
+ {
1032
+ "loss": 0.7184643268585205,
1033
+ "grad_norm": 2.0378243923187256,
1034
+ "learning_rate": 0.00018817856251549867,
1035
+ "entropy": 0.7140736062079668,
1036
+ "num_tokens": 8219594.0,
1037
+ "mean_token_accuracy": 0.7989312529563903,
1038
+ "epoch": 1.9890547263681593,
1039
+ "step": 1000
1040
+ },
1041
+ {
1042
+ "eval_loss": 1.0154445171356201,
1043
+ "eval_runtime": 88.2,
1044
+ "eval_samples_per_second": 11.735,
1045
+ "eval_steps_per_second": 5.873,
1046
+ "eval_entropy": 0.8002531397008988,
1047
+ "eval_num_tokens": 8265312.0,
1048
+ "eval_mean_token_accuracy": 0.7423216365709268,
1049
+ "epoch": 2.0,
1050
+ "step": 1006
1051
+ },
1052
+ {
1053
+ "loss": 0.6638185501098632,
1054
+ "grad_norm": 1.6040326356887817,
1055
+ "learning_rate": 0.00018786654284258034,
1056
+ "entropy": 0.7038449613671554,
1057
+ "num_tokens": 8297623.0,
1058
+ "mean_token_accuracy": 0.8091712115626586,
1059
+ "epoch": 2.007960199004975,
1060
+ "step": 1010
1061
+ },
1062
+ {
1063
+ "loss": 0.5153174877166748,
1064
+ "grad_norm": 2.1827661991119385,
1065
+ "learning_rate": 0.00018755072451836097,
1066
+ "entropy": 0.5252644840627909,
1067
+ "num_tokens": 8379050.0,
1068
+ "mean_token_accuracy": 0.8446948520839215,
1069
+ "epoch": 2.027860696517413,
1070
+ "step": 1020
1071
+ },
1072
+ {
1073
+ "loss": 0.5017234325408936,
1074
+ "grad_norm": 1.8835452795028687,
1075
+ "learning_rate": 0.00018723112119631608,
1076
+ "entropy": 0.5121854435652494,
1077
+ "num_tokens": 8462847.0,
1078
+ "mean_token_accuracy": 0.8539764225482941,
1079
+ "epoch": 2.047761194029851,
1080
+ "step": 1030
1081
+ },
1082
+ {
1083
+ "loss": 0.5333661079406739,
1084
+ "grad_norm": 1.6592763662338257,
1085
+ "learning_rate": 0.00018690774669355442,
1086
+ "entropy": 0.5651770515367389,
1087
+ "num_tokens": 8542681.0,
1088
+ "mean_token_accuracy": 0.8419696733355522,
1089
+ "epoch": 2.0676616915422885,
1090
+ "step": 1040
1091
+ },
1092
+ {
1093
+ "loss": 0.4740941047668457,
1094
+ "grad_norm": 1.8748646974563599,
1095
+ "learning_rate": 0.00018658061499022055,
1096
+ "entropy": 0.47107693143188956,
1097
+ "num_tokens": 8627432.0,
1098
+ "mean_token_accuracy": 0.8567564368247986,
1099
+ "epoch": 2.0875621890547262,
1100
+ "step": 1050
1101
+ },
1102
+ {
1103
+ "loss": 0.48979806900024414,
1104
+ "grad_norm": 2.027717351913452,
1105
+ "learning_rate": 0.0001862497402288906,
1106
+ "entropy": 0.49270418751984835,
1107
+ "num_tokens": 8708822.0,
1108
+ "mean_token_accuracy": 0.8501696541905404,
1109
+ "epoch": 2.107462686567164,
1110
+ "step": 1060
1111
+ },
1112
+ {
1113
+ "loss": 0.4521069049835205,
1114
+ "grad_norm": 1.8452632427215576,
1115
+ "learning_rate": 0.0001859151367139608,
1116
+ "entropy": 0.4634111661463976,
1117
+ "num_tokens": 8789791.0,
1118
+ "mean_token_accuracy": 0.8683336563408375,
1119
+ "epoch": 2.127363184079602,
1120
+ "step": 1070
1121
+ },
1122
+ {
1123
+ "loss": 0.5178594589233398,
1124
+ "grad_norm": 1.9295680522918701,
1125
+ "learning_rate": 0.000185576818911029,
1126
+ "entropy": 0.46942942440509794,
1127
+ "num_tokens": 8870514.0,
1128
+ "mean_token_accuracy": 0.8462874710559845,
1129
+ "epoch": 2.14726368159204,
1130
+ "step": 1080
1131
+ },
1132
+ {
1133
+ "loss": 0.5196755409240723,
1134
+ "grad_norm": 1.6308494806289673,
1135
+ "learning_rate": 0.00018523480144626948,
1136
+ "entropy": 0.5393101962283253,
1137
+ "num_tokens": 8952466.0,
1138
+ "mean_token_accuracy": 0.8491615362465381,
1139
+ "epoch": 2.1671641791044776,
1140
+ "step": 1090
1141
+ },
1142
+ {
1143
+ "loss": 0.507460069656372,
1144
+ "grad_norm": 1.9126944541931152,
1145
+ "learning_rate": 0.00018488909910580037,
1146
+ "entropy": 0.491584631241858,
1147
+ "num_tokens": 9034965.0,
1148
+ "mean_token_accuracy": 0.8491294652223587,
1149
+ "epoch": 2.1870646766169153,
1150
+ "step": 1100
1151
+ },
1152
+ {
1153
+ "loss": 0.5434001445770263,
1154
+ "grad_norm": 2.245730400085449,
1155
+ "learning_rate": 0.00018453972683504466,
1156
+ "entropy": 0.5460982380434871,
1157
+ "num_tokens": 9117636.0,
1158
+ "mean_token_accuracy": 0.8417311415076256,
1159
+ "epoch": 2.206965174129353,
1160
+ "step": 1110
1161
+ },
1162
+ {
1163
+ "loss": 0.5176630973815918,
1164
+ "grad_norm": 1.7490206956863403,
1165
+ "learning_rate": 0.00018418669973808386,
1166
+ "entropy": 0.5413472417742013,
1167
+ "num_tokens": 9203719.0,
1168
+ "mean_token_accuracy": 0.8437662713229657,
1169
+ "epoch": 2.226865671641791,
1170
+ "step": 1120
1171
+ },
1172
+ {
1173
+ "loss": 0.4966591358184814,
1174
+ "grad_norm": 2.32169246673584,
1175
+ "learning_rate": 0.00018383003307700525,
1176
+ "entropy": 0.5068975739181042,
1177
+ "num_tokens": 9285285.0,
1178
+ "mean_token_accuracy": 0.8558630302548409,
1179
+ "epoch": 2.246766169154229,
1180
+ "step": 1130
1181
+ },
1182
+ {
1183
+ "loss": 0.4562082290649414,
1184
+ "grad_norm": 2.121119499206543,
1185
+ "learning_rate": 0.0001834697422712419,
1186
+ "entropy": 0.472692010179162,
1187
+ "num_tokens": 9368724.0,
1188
+ "mean_token_accuracy": 0.861832109093666,
1189
+ "epoch": 2.2666666666666666,
1190
+ "step": 1140
1191
+ },
1192
+ {
1193
+ "loss": 0.5065945148468017,
1194
+ "grad_norm": 2.003596305847168,
1195
+ "learning_rate": 0.00018310584289690608,
1196
+ "entropy": 0.48463920764625074,
1197
+ "num_tokens": 9450345.0,
1198
+ "mean_token_accuracy": 0.8411129087209701,
1199
+ "epoch": 2.2865671641791043,
1200
+ "step": 1150
1201
+ },
1202
+ {
1203
+ "loss": 0.5093509674072265,
1204
+ "grad_norm": 1.9170504808425903,
1205
+ "learning_rate": 0.0001827383506861159,
1206
+ "entropy": 0.5297361209988594,
1207
+ "num_tokens": 9531914.0,
1208
+ "mean_token_accuracy": 0.8439007833600044,
1209
+ "epoch": 2.306467661691542,
1210
+ "step": 1160
1211
+ },
1212
+ {
1213
+ "loss": 0.5065027236938476,
1214
+ "grad_norm": 1.8182140588760376,
1215
+ "learning_rate": 0.00018236728152631526,
1216
+ "entropy": 0.508120141364634,
1217
+ "num_tokens": 9613824.0,
1218
+ "mean_token_accuracy": 0.8519511334598064,
1219
+ "epoch": 2.32636815920398,
1220
+ "step": 1170
1221
+ },
1222
+ {
1223
+ "loss": 0.5736330032348633,
1224
+ "grad_norm": 1.6451319456100464,
1225
+ "learning_rate": 0.00018199265145958678,
1226
+ "entropy": 0.5490097716450691,
1227
+ "num_tokens": 9696070.0,
1228
+ "mean_token_accuracy": 0.8286732420325279,
1229
+ "epoch": 2.346268656716418,
1230
+ "step": 1180
1231
+ },
1232
+ {
1233
+ "loss": 0.5310076236724853,
1234
+ "grad_norm": 1.8924293518066406,
1235
+ "learning_rate": 0.00018161447668195858,
1236
+ "entropy": 0.5155842589214444,
1237
+ "num_tokens": 9777568.0,
1238
+ "mean_token_accuracy": 0.8431366585195065,
1239
+ "epoch": 2.3661691542288557,
1240
+ "step": 1190
1241
+ },
1242
+ {
1243
+ "loss": 0.46693716049194334,
1244
+ "grad_norm": 1.9782637357711792,
1245
+ "learning_rate": 0.00018123277354270372,
1246
+ "entropy": 0.4998965173959732,
1247
+ "num_tokens": 9858081.0,
1248
+ "mean_token_accuracy": 0.8578181132674217,
1249
+ "epoch": 2.3860696517412934,
1250
+ "step": 1200
1251
+ },
1252
+ {
1253
+ "loss": 0.5384686946868896,
1254
+ "grad_norm": 1.8333914279937744,
1255
+ "learning_rate": 0.00018084755854363377,
1256
+ "entropy": 0.5187313890084624,
1257
+ "num_tokens": 9940187.0,
1258
+ "mean_token_accuracy": 0.8451795615255833,
1259
+ "epoch": 2.405970149253731,
1260
+ "step": 1210
1261
+ },
1262
+ {
1263
+ "loss": 0.5079335689544677,
1264
+ "grad_norm": 2.00839900970459,
1265
+ "learning_rate": 0.00018045884833838512,
1266
+ "entropy": 0.5085030103102326,
1267
+ "num_tokens": 10018432.0,
1268
+ "mean_token_accuracy": 0.8514300569891929,
1269
+ "epoch": 2.4258706467661693,
1270
+ "step": 1220
1271
+ },
1272
+ {
1273
+ "loss": 0.5386298179626465,
1274
+ "grad_norm": 1.9003617763519287,
1275
+ "learning_rate": 0.00018006665973169911,
1276
+ "entropy": 0.5382265558466315,
1277
+ "num_tokens": 10100137.0,
1278
+ "mean_token_accuracy": 0.8454402416944504,
1279
+ "epoch": 2.445771144278607,
1280
+ "step": 1230
1281
+ },
1282
+ {
1283
+ "loss": 0.5553887367248536,
1284
+ "grad_norm": 2.1470773220062256,
1285
+ "learning_rate": 0.0001796710096786956,
1286
+ "entropy": 0.5438722034916281,
1287
+ "num_tokens": 10184025.0,
1288
+ "mean_token_accuracy": 0.8391343042254448,
1289
+ "epoch": 2.4656716417910447,
1290
+ "step": 1240
1291
+ },
1292
+ {
1293
+ "loss": 0.5232258319854737,
1294
+ "grad_norm": 1.992197036743164,
1295
+ "learning_rate": 0.0001792719152841398,
1296
+ "entropy": 0.5251597924157977,
1297
+ "num_tokens": 10268664.0,
1298
+ "mean_token_accuracy": 0.8480332940816879,
1299
+ "epoch": 2.4855721393034824,
1300
+ "step": 1250
1301
+ },
1302
+ {
1303
+ "loss": 0.5130891799926758,
1304
+ "grad_norm": 1.6610969305038452,
1305
+ "learning_rate": 0.0001788693938017029,
1306
+ "entropy": 0.5361188564449548,
1307
+ "num_tokens": 10349492.0,
1308
+ "mean_token_accuracy": 0.8441365607082844,
1309
+ "epoch": 2.50547263681592,
1310
+ "step": 1260
1311
+ },
1312
+ {
1313
+ "loss": 0.5028162956237793,
1314
+ "grad_norm": 2.0173373222351074,
1315
+ "learning_rate": 0.00017846346263321623,
1316
+ "entropy": 0.4794240856543183,
1317
+ "num_tokens": 10432597.0,
1318
+ "mean_token_accuracy": 0.84928208142519,
1319
+ "epoch": 2.5253731343283583,
1320
+ "step": 1270
1321
+ },
1322
+ {
1323
+ "loss": 0.5137399673461914,
1324
+ "grad_norm": 2.124582290649414,
1325
+ "learning_rate": 0.00017805413932791875,
1326
+ "entropy": 0.5074908941984176,
1327
+ "num_tokens": 10513192.0,
1328
+ "mean_token_accuracy": 0.8484948351979256,
1329
+ "epoch": 2.545273631840796,
1330
+ "step": 1280
1331
+ },
1332
+ {
1333
+ "loss": 0.5218567848205566,
1334
+ "grad_norm": 1.7741371393203735,
1335
+ "learning_rate": 0.00017764144158169856,
1336
+ "entropy": 0.5202818088233471,
1337
+ "num_tokens": 10597049.0,
1338
+ "mean_token_accuracy": 0.8525593280792236,
1339
+ "epoch": 2.5651741293532337,
1340
+ "step": 1290
1341
+ },
1342
+ {
1343
+ "loss": 0.5342438697814942,
1344
+ "grad_norm": 2.03674054145813,
1345
+ "learning_rate": 0.00017722538723632773,
1346
+ "entropy": 0.5169114828109741,
1347
+ "num_tokens": 10679839.0,
1348
+ "mean_token_accuracy": 0.8379098229110241,
1349
+ "epoch": 2.585074626865672,
1350
+ "step": 1300
1351
+ },
1352
+ {
1353
+ "loss": 0.5268249988555909,
1354
+ "grad_norm": 1.7481690645217896,
1355
+ "learning_rate": 0.000176805994278691,
1356
+ "entropy": 0.527920619212091,
1357
+ "num_tokens": 10763374.0,
1358
+ "mean_token_accuracy": 0.8468565516173839,
1359
+ "epoch": 2.604975124378109,
1360
+ "step": 1310
1361
+ },
1362
+ {
1363
+ "loss": 0.5637305736541748,
1364
+ "grad_norm": 2.044337511062622,
1365
+ "learning_rate": 0.0001763832808400082,
1366
+ "entropy": 0.5612680882215499,
1367
+ "num_tokens": 10846467.0,
1368
+ "mean_token_accuracy": 0.8389740340411663,
1369
+ "epoch": 2.6248756218905474,
1370
+ "step": 1320
1371
+ },
1372
+ {
1373
+ "loss": 0.5200609683990478,
1374
+ "grad_norm": 1.5558581352233887,
1375
+ "learning_rate": 0.00017595726519505043,
1376
+ "entropy": 0.526572679169476,
1377
+ "num_tokens": 10929729.0,
1378
+ "mean_token_accuracy": 0.848398495465517,
1379
+ "epoch": 2.644776119402985,
1380
+ "step": 1330
1381
+ },
1382
+ {
1383
+ "loss": 0.5190268993377686,
1384
+ "grad_norm": 2.233233690261841,
1385
+ "learning_rate": 0.00017552796576134985,
1386
+ "entropy": 0.5076324449852109,
1387
+ "num_tokens": 11012769.0,
1388
+ "mean_token_accuracy": 0.841417095810175,
1389
+ "epoch": 2.664676616915423,
1390
+ "step": 1340
1391
+ },
1392
+ {
1393
+ "loss": 0.547359848022461,
1394
+ "grad_norm": 1.8738890886306763,
1395
+ "learning_rate": 0.00017509540109840365,
1396
+ "entropy": 0.5502721823751926,
1397
+ "num_tokens": 11094716.0,
1398
+ "mean_token_accuracy": 0.8380319178104401,
1399
+ "epoch": 2.684577114427861,
1400
+ "step": 1350
1401
+ },
1402
+ {
1403
+ "loss": 0.5218305110931396,
1404
+ "grad_norm": 1.8149367570877075,
1405
+ "learning_rate": 0.00017465958990687156,
1406
+ "entropy": 0.5275072449818253,
1407
+ "num_tokens": 11175013.0,
1408
+ "mean_token_accuracy": 0.8568139627575875,
1409
+ "epoch": 2.7044776119402982,
1410
+ "step": 1360
1411
+ },
1412
+ {
1413
+ "loss": 0.5018157482147216,
1414
+ "grad_norm": 1.698498010635376,
1415
+ "learning_rate": 0.0001742205510277674,
1416
+ "entropy": 0.5119684131816029,
1417
+ "num_tokens": 11256288.0,
1418
+ "mean_token_accuracy": 0.8495248600840568,
1419
+ "epoch": 2.7243781094527364,
1420
+ "step": 1370
1421
+ },
1422
+ {
1423
+ "loss": 0.5008152484893799,
1424
+ "grad_norm": 1.837099552154541,
1425
+ "learning_rate": 0.00017377830344164464,
1426
+ "entropy": 0.5109101135283709,
1427
+ "num_tokens": 11338498.0,
1428
+ "mean_token_accuracy": 0.8569738574326038,
1429
+ "epoch": 2.744278606965174,
1430
+ "step": 1380
1431
+ },
1432
+ {
1433
+ "loss": 0.5308613777160645,
1434
+ "grad_norm": 2.1258440017700195,
1435
+ "learning_rate": 0.00017333286626777564,
1436
+ "entropy": 0.5329740956425667,
1437
+ "num_tokens": 11422122.0,
1438
+ "mean_token_accuracy": 0.8408495858311653,
1439
+ "epoch": 2.764179104477612,
1440
+ "step": 1390
1441
+ },
1442
+ {
1443
+ "loss": 0.5461023807525635,
1444
+ "grad_norm": 2.0416526794433594,
1445
+ "learning_rate": 0.00017288425876332527,
1446
+ "entropy": 0.5458354912698269,
1447
+ "num_tokens": 11506227.0,
1448
+ "mean_token_accuracy": 0.8418126352131367,
1449
+ "epoch": 2.78407960199005,
1450
+ "step": 1400
1451
+ },
1452
+ {
1453
+ "loss": 0.5320337772369385,
1454
+ "grad_norm": 1.8857133388519287,
1455
+ "learning_rate": 0.00017243250032251823,
1456
+ "entropy": 0.5177356921136379,
1457
+ "num_tokens": 11586611.0,
1458
+ "mean_token_accuracy": 0.8437633819878101,
1459
+ "epoch": 2.8039800995024877,
1460
+ "step": 1410
1461
+ },
1462
+ {
1463
+ "loss": 0.4947951793670654,
1464
+ "grad_norm": 2.1604552268981934,
1465
+ "learning_rate": 0.00017197761047580082,
1466
+ "entropy": 0.5068972071632742,
1467
+ "num_tokens": 11670179.0,
1468
+ "mean_token_accuracy": 0.8533644363284111,
1469
+ "epoch": 2.8238805970149254,
1470
+ "step": 1420
1471
+ },
1472
+ {
1473
+ "loss": 0.5062966346740723,
1474
+ "grad_norm": 1.9001272916793823,
1475
+ "learning_rate": 0.00017151960888899627,
1476
+ "entropy": 0.491135448589921,
1477
+ "num_tokens": 11750106.0,
1478
+ "mean_token_accuracy": 0.8556574188172817,
1479
+ "epoch": 2.843781094527363,
1480
+ "step": 1430
1481
+ },
1482
+ {
1483
+ "loss": 0.48212461471557616,
1484
+ "grad_norm": 1.9398213624954224,
1485
+ "learning_rate": 0.00017105851536245492,
1486
+ "entropy": 0.5211849508807063,
1487
+ "num_tokens": 11831237.0,
1488
+ "mean_token_accuracy": 0.8569056294858456,
1489
+ "epoch": 2.863681592039801,
1490
+ "step": 1440
1491
+ },
1492
+ {
1493
+ "loss": 0.5234591960906982,
1494
+ "grad_norm": 1.8992606401443481,
1495
+ "learning_rate": 0.0001705943498301979,
1496
+ "entropy": 0.4800686825066805,
1497
+ "num_tokens": 11915761.0,
1498
+ "mean_token_accuracy": 0.8473109625279903,
1499
+ "epoch": 2.883582089552239,
1500
+ "step": 1450
1501
+ },
1502
+ {
1503
+ "loss": 0.5000662803649902,
1504
+ "grad_norm": 1.7360588312149048,
1505
+ "learning_rate": 0.00017012713235905547,
1506
+ "entropy": 0.5110673246905207,
1507
+ "num_tokens": 11996666.0,
1508
+ "mean_token_accuracy": 0.8583845689892768,
1509
+ "epoch": 2.9034825870646768,
1510
+ "step": 1460
1511
+ },
1512
+ {
1513
+ "loss": 0.5083964824676513,
1514
+ "grad_norm": 2.0405280590057373,
1515
+ "learning_rate": 0.00016965688314779956,
1516
+ "entropy": 0.49351408518850803,
1517
+ "num_tokens": 12082260.0,
1518
+ "mean_token_accuracy": 0.8502242051064968,
1519
+ "epoch": 2.9233830845771145,
1520
+ "step": 1470
1521
+ },
1522
+ {
1523
+ "loss": 0.5238072872161865,
1524
+ "grad_norm": 1.9192203283309937,
1525
+ "learning_rate": 0.00016918362252627036,
1526
+ "entropy": 0.5278301501646638,
1527
+ "num_tokens": 12160973.0,
1528
+ "mean_token_accuracy": 0.8439404472708703,
1529
+ "epoch": 2.943283582089552,
1530
+ "step": 1480
1531
+ },
1532
+ {
1533
+ "loss": 0.5171586036682129,
1534
+ "grad_norm": 1.6478546857833862,
1535
+ "learning_rate": 0.00016870737095449754,
1536
+ "entropy": 0.5115002764388918,
1537
+ "num_tokens": 12244781.0,
1538
+ "mean_token_accuracy": 0.8549239777028561,
1539
+ "epoch": 2.96318407960199,
1540
+ "step": 1490
1541
+ },
1542
+ {
1543
+ "loss": 0.4724470615386963,
1544
+ "grad_norm": 2.152078866958618,
1545
+ "learning_rate": 0.00016822814902181583,
1546
+ "entropy": 0.4658357990905643,
1547
+ "num_tokens": 12328430.0,
1548
+ "mean_token_accuracy": 0.8588747374713421,
1549
+ "epoch": 2.983084577114428,
1550
+ "step": 1500
1551
+ },
1552
+ {
1553
+ "eval_loss": 1.013381004333496,
1554
+ "eval_runtime": 89.0725,
1555
+ "eval_samples_per_second": 11.62,
1556
+ "eval_steps_per_second": 5.815,
1557
+ "eval_entropy": 0.6918873670421052,
1558
+ "eval_num_tokens": 12397968.0,
1559
+ "eval_mean_token_accuracy": 0.7516806636537824,
1560
+ "epoch": 3.0,
1561
+ "step": 1509
1562
+ },
1563
+ {
1564
+ "loss": 0.524122667312622,
1565
+ "grad_norm": 1.2526023387908936,
1566
+ "learning_rate": 0.00016774597744597457,
1567
+ "entropy": 0.5304617995494291,
1568
+ "num_tokens": 12405849.0,
1569
+ "mean_token_accuracy": 0.841843033307477,
1570
+ "epoch": 3.0019900497512437,
1571
+ "step": 1510
1572
+ },
1573
+ {
1574
+ "loss": 0.3169110774993896,
1575
+ "grad_norm": 2.0134313106536865,
1576
+ "learning_rate": 0.00016726087707224236,
1577
+ "entropy": 0.37756410874426366,
1578
+ "num_tokens": 12487545.0,
1579
+ "mean_token_accuracy": 0.9008184991776943,
1580
+ "epoch": 3.0218905472636814,
1581
+ "step": 1520
1582
+ },
1583
+ {
1584
+ "loss": 0.31904311180114747,
1585
+ "grad_norm": 1.9329195022583008,
1586
+ "learning_rate": 0.00016677286887250572,
1587
+ "entropy": 0.31940033063292506,
1588
+ "num_tokens": 12569892.0,
1589
+ "mean_token_accuracy": 0.8977621793746948,
1590
+ "epoch": 3.0417910447761196,
1591
+ "step": 1530
1592
+ },
1593
+ {
1594
+ "loss": 0.31394209861755373,
1595
+ "grad_norm": 1.681523323059082,
1596
+ "learning_rate": 0.00016628197394436247,
1597
+ "entropy": 0.34449700023978946,
1598
+ "num_tokens": 12651094.0,
1599
+ "mean_token_accuracy": 0.9006784312427044,
1600
+ "epoch": 3.0616915422885573,
1601
+ "step": 1540
1602
+ },
1603
+ {
1604
+ "loss": 0.29366445541381836,
1605
+ "grad_norm": 1.9879311323165894,
1606
+ "learning_rate": 0.00016578821351020964,
1607
+ "entropy": 0.315945752710104,
1608
+ "num_tokens": 12732906.0,
1609
+ "mean_token_accuracy": 0.9038687475025654,
1610
+ "epoch": 3.081592039800995,
1611
+ "step": 1550
1612
+ },
1613
+ {
1614
+ "loss": 0.32794480323791503,
1615
+ "grad_norm": 1.9403680562973022,
1616
+ "learning_rate": 0.00016529160891632597,
1617
+ "entropy": 0.3383891684934497,
1618
+ "num_tokens": 12814067.0,
1619
+ "mean_token_accuracy": 0.893728756159544,
1620
+ "epoch": 3.1014925373134328,
1621
+ "step": 1560
1622
+ },
1623
+ {
1624
+ "loss": 0.3342399597167969,
1625
+ "grad_norm": 1.9990283250808716,
1626
+ "learning_rate": 0.00016479218163194904,
1627
+ "entropy": 0.33618702124804256,
1628
+ "num_tokens": 12895605.0,
1629
+ "mean_token_accuracy": 0.8921422258019447,
1630
+ "epoch": 3.1213930348258705,
1631
+ "step": 1570
1632
+ },
1633
+ {
1634
+ "loss": 0.32936697006225585,
1635
+ "grad_norm": 1.9695919752120972,
1636
+ "learning_rate": 0.00016428995324834723,
1637
+ "entropy": 0.3401200842112303,
1638
+ "num_tokens": 12979094.0,
1639
+ "mean_token_accuracy": 0.9001650154590607,
1640
+ "epoch": 3.1412935323383087,
1641
+ "step": 1580
1642
+ },
1643
+ {
1644
+ "loss": 0.30141172409057615,
1645
+ "grad_norm": 1.8548579216003418,
1646
+ "learning_rate": 0.00016378494547788613,
1647
+ "entropy": 0.3256719596683979,
1648
+ "num_tokens": 13061107.0,
1649
+ "mean_token_accuracy": 0.9027377247810364,
1650
+ "epoch": 3.1611940298507464,
1651
+ "step": 1590
1652
+ },
1653
+ {
1654
+ "loss": 0.33073742389678956,
1655
+ "grad_norm": 2.56424617767334,
1656
+ "learning_rate": 0.00016327718015308998,
1657
+ "entropy": 0.33555619083344934,
1658
+ "num_tokens": 13143766.0,
1659
+ "mean_token_accuracy": 0.8924005568027497,
1660
+ "epoch": 3.181094527363184,
1661
+ "step": 1600
1662
+ },
1663
+ {
1664
+ "loss": 0.3116974592208862,
1665
+ "grad_norm": 1.8065255880355835,
1666
+ "learning_rate": 0.0001627666792256977,
1667
+ "entropy": 0.3388229014351964,
1668
+ "num_tokens": 13228296.0,
1669
+ "mean_token_accuracy": 0.9034810848534107,
1670
+ "epoch": 3.200995024875622,
1671
+ "step": 1610
1672
+ },
1673
+ {
1674
+ "loss": 0.35255770683288573,
1675
+ "grad_norm": 2.3625452518463135,
1676
+ "learning_rate": 0.00016225346476571396,
1677
+ "entropy": 0.35893154982477427,
1678
+ "num_tokens": 13310326.0,
1679
+ "mean_token_accuracy": 0.8877448745071888,
1680
+ "epoch": 3.2208955223880595,
1681
+ "step": 1620
1682
+ },
1683
+ {
1684
+ "loss": 0.35570693016052246,
1685
+ "grad_norm": 2.29675030708313,
1686
+ "learning_rate": 0.00016173755896045506,
1687
+ "entropy": 0.3679086913354695,
1688
+ "num_tokens": 13391862.0,
1689
+ "mean_token_accuracy": 0.8964587457478046,
1690
+ "epoch": 3.2407960199004977,
1691
+ "step": 1630
1692
+ },
1693
+ {
1694
+ "loss": 0.35628087520599366,
1695
+ "grad_norm": 1.8130031824111938,
1696
+ "learning_rate": 0.00016121898411358966,
1697
+ "entropy": 0.36298612505197525,
1698
+ "num_tokens": 13476890.0,
1699
+ "mean_token_accuracy": 0.8880530841648578,
1700
+ "epoch": 3.2606965174129354,
1701
+ "step": 1640
1702
+ },
1703
+ {
1704
+ "loss": 0.3256735324859619,
1705
+ "grad_norm": 1.7297886610031128,
1706
+ "learning_rate": 0.00016069776264417463,
1707
+ "entropy": 0.3571936976164579,
1708
+ "num_tokens": 13560962.0,
1709
+ "mean_token_accuracy": 0.8932363368570805,
1710
+ "epoch": 3.280597014925373,
1711
+ "step": 1650
1712
+ },
1713
+ {
1714
+ "loss": 0.3441990852355957,
1715
+ "grad_norm": 2.114352226257324,
1716
+ "learning_rate": 0.00016017391708568563,
1717
+ "entropy": 0.33980772607028487,
1718
+ "num_tokens": 13642374.0,
1719
+ "mean_token_accuracy": 0.8990235075354576,
1720
+ "epoch": 3.300497512437811,
1721
+ "step": 1660
1722
+ },
1723
+ {
1724
+ "loss": 0.3780921697616577,
1725
+ "grad_norm": 2.2089595794677734,
1726
+ "learning_rate": 0.00015964747008504325,
1727
+ "entropy": 0.3775214830413461,
1728
+ "num_tokens": 13724787.0,
1729
+ "mean_token_accuracy": 0.8860153228044509,
1730
+ "epoch": 3.3203980099502486,
1731
+ "step": 1670
1732
+ },
1733
+ {
1734
+ "loss": 0.35315823554992676,
1735
+ "grad_norm": 1.8260467052459717,
1736
+ "learning_rate": 0.00015911844440163371,
1737
+ "entropy": 0.37848529405891895,
1738
+ "num_tokens": 13804379.0,
1739
+ "mean_token_accuracy": 0.8853729166090488,
1740
+ "epoch": 3.3402985074626868,
1741
+ "step": 1680
1742
+ },
1743
+ {
1744
+ "loss": 0.3457970142364502,
1745
+ "grad_norm": 2.5162901878356934,
1746
+ "learning_rate": 0.00015858686290632493,
1747
+ "entropy": 0.35254298616200686,
1748
+ "num_tokens": 13883828.0,
1749
+ "mean_token_accuracy": 0.8893432796001435,
1750
+ "epoch": 3.3601990049751245,
1751
+ "step": 1690
1752
+ },
1753
+ {
1754
+ "loss": 0.3127096176147461,
1755
+ "grad_norm": 2.613616466522217,
1756
+ "learning_rate": 0.0001580527485804779,
1757
+ "entropy": 0.3147669959813356,
1758
+ "num_tokens": 13965784.0,
1759
+ "mean_token_accuracy": 0.903999711573124,
1760
+ "epoch": 3.380099502487562,
1761
+ "step": 1700
1762
+ },
1763
+ {
1764
+ "loss": 0.338837718963623,
1765
+ "grad_norm": 1.8569570779800415,
1766
+ "learning_rate": 0.00015751612451495313,
1767
+ "entropy": 0.345753663033247,
1768
+ "num_tokens": 14048546.0,
1769
+ "mean_token_accuracy": 0.896949079632759,
1770
+ "epoch": 3.4,
1771
+ "step": 1710
1772
+ },
1773
+ {
1774
+ "loss": 0.3418254852294922,
1775
+ "grad_norm": 1.975127100944519,
1776
+ "learning_rate": 0.00015697701390911218,
1777
+ "entropy": 0.3369439946487546,
1778
+ "num_tokens": 14130348.0,
1779
+ "mean_token_accuracy": 0.8862252190709115,
1780
+ "epoch": 3.4199004975124376,
1781
+ "step": 1720
1782
+ },
1783
+ {
1784
+ "loss": 0.36044361591339114,
1785
+ "grad_norm": 1.8558331727981567,
1786
+ "learning_rate": 0.00015643544006981505,
1787
+ "entropy": 0.3699260259047151,
1788
+ "num_tokens": 14209009.0,
1789
+ "mean_token_accuracy": 0.8909381292760372,
1790
+ "epoch": 3.439800995024876,
1791
+ "step": 1730
1792
+ },
1793
+ {
1794
+ "loss": 0.31586222648620604,
1795
+ "grad_norm": 1.7793047428131104,
1796
+ "learning_rate": 0.0001558914264104122,
1797
+ "entropy": 0.3264750910922885,
1798
+ "num_tokens": 14289936.0,
1799
+ "mean_token_accuracy": 0.8986882567405701,
1800
+ "epoch": 3.4597014925373135,
1801
+ "step": 1740
1802
+ },
1803
+ {
1804
+ "loss": 0.3266884803771973,
1805
+ "grad_norm": 1.7176367044448853,
1806
+ "learning_rate": 0.00015534499644973267,
1807
+ "entropy": 0.3284300332888961,
1808
+ "num_tokens": 14372769.0,
1809
+ "mean_token_accuracy": 0.8963496647775173,
1810
+ "epoch": 3.4796019900497512,
1811
+ "step": 1750
1812
+ },
1813
+ {
1814
+ "loss": 0.31137449741363527,
1815
+ "grad_norm": 2.04630184173584,
1816
+ "learning_rate": 0.00015479617381106711,
1817
+ "entropy": 0.34350175261497495,
1818
+ "num_tokens": 14456141.0,
1819
+ "mean_token_accuracy": 0.9009241946041584,
1820
+ "epoch": 3.499502487562189,
1821
+ "step": 1760
1822
+ },
1823
+ {
1824
+ "loss": 0.3356631755828857,
1825
+ "grad_norm": 1.8468575477600098,
1826
+ "learning_rate": 0.00015424498222114662,
1827
+ "entropy": 0.3304918609559536,
1828
+ "num_tokens": 14540151.0,
1829
+ "mean_token_accuracy": 0.8924445942044258,
1830
+ "epoch": 3.5194029850746267,
1831
+ "step": 1770
1832
+ },
1833
+ {
1834
+ "loss": 0.3501142501831055,
1835
+ "grad_norm": 1.9734855890274048,
1836
+ "learning_rate": 0.00015369144550911678,
1837
+ "entropy": 0.36590540781617165,
1838
+ "num_tokens": 14624972.0,
1839
+ "mean_token_accuracy": 0.88669129088521,
1840
+ "epoch": 3.539303482587065,
1841
+ "step": 1780
1842
+ },
1843
+ {
1844
+ "loss": 0.3299347639083862,
1845
+ "grad_norm": 2.0176360607147217,
1846
+ "learning_rate": 0.0001531355876055078,
1847
+ "entropy": 0.33581888526678083,
1848
+ "num_tokens": 14707098.0,
1849
+ "mean_token_accuracy": 0.8888270042836666,
1850
+ "epoch": 3.5592039800995026,
1851
+ "step": 1790
1852
+ },
1853
+ {
1854
+ "loss": 0.3689806222915649,
1855
+ "grad_norm": 2.206531047821045,
1856
+ "learning_rate": 0.00015257743254119973,
1857
+ "entropy": 0.3515732469037175,
1858
+ "num_tokens": 14789628.0,
1859
+ "mean_token_accuracy": 0.8852153360843659,
1860
+ "epoch": 3.5791044776119403,
1861
+ "step": 1800
1862
+ },
1863
+ {
1864
+ "loss": 0.32832067012786864,
1865
+ "grad_norm": 1.6956323385238647,
1866
+ "learning_rate": 0.00015201700444638348,
1867
+ "entropy": 0.35302160643041136,
1868
+ "num_tokens": 14870207.0,
1869
+ "mean_token_accuracy": 0.8937827706336975,
1870
+ "epoch": 3.599004975124378,
1871
+ "step": 1810
1872
+ },
1873
+ {
1874
+ "loss": 0.37772986888885496,
1875
+ "grad_norm": 1.7329686880111694,
1876
+ "learning_rate": 0.00015145432754951784,
1877
+ "entropy": 0.3665313167497516,
1878
+ "num_tokens": 14952333.0,
1879
+ "mean_token_accuracy": 0.8815502606332302,
1880
+ "epoch": 3.6189054726368157,
1881
+ "step": 1820
1882
+ },
1883
+ {
1884
+ "loss": 0.3516186237335205,
1885
+ "grad_norm": 1.77907133102417,
1886
+ "learning_rate": 0.000150889426176282,
1887
+ "entropy": 0.3640569668263197,
1888
+ "num_tokens": 15033429.0,
1889
+ "mean_token_accuracy": 0.8912140592932701,
1890
+ "epoch": 3.638805970149254,
1891
+ "step": 1830
1892
+ },
1893
+ {
1894
+ "loss": 0.37269628047943115,
1895
+ "grad_norm": 1.7633020877838135,
1896
+ "learning_rate": 0.00015032232474852371,
1897
+ "entropy": 0.3742990693077445,
1898
+ "num_tokens": 15113657.0,
1899
+ "mean_token_accuracy": 0.8849355317652225,
1900
+ "epoch": 3.6587064676616916,
1901
+ "step": 1840
1902
+ },
1903
+ {
1904
+ "loss": 0.3090696096420288,
1905
+ "grad_norm": 1.9857285022735596,
1906
+ "learning_rate": 0.00014975304778320364,
1907
+ "entropy": 0.3482851915061474,
1908
+ "num_tokens": 15197089.0,
1909
+ "mean_token_accuracy": 0.9004555702209472,
1910
+ "epoch": 3.6786069651741293,
1911
+ "step": 1850
1912
+ },
1913
+ {
1914
+ "loss": 0.36194648742675783,
1915
+ "grad_norm": 2.1343369483947754,
1916
+ "learning_rate": 0.00014918161989133552,
1917
+ "entropy": 0.3528735195286572,
1918
+ "num_tokens": 15283755.0,
1919
+ "mean_token_accuracy": 0.8829138934612274,
1920
+ "epoch": 3.698507462686567,
1921
+ "step": 1860
1922
+ },
1923
+ {
1924
+ "loss": 0.3398463010787964,
1925
+ "grad_norm": 1.8817847967147827,
1926
+ "learning_rate": 0.0001486080657769219,
1927
+ "entropy": 0.3488534286618233,
1928
+ "num_tokens": 15367535.0,
1929
+ "mean_token_accuracy": 0.896855903416872,
1930
+ "epoch": 3.7184079601990048,
1931
+ "step": 1870
1932
+ },
1933
+ {
1934
+ "loss": 0.37033088207244874,
1935
+ "grad_norm": 1.852171778678894,
1936
+ "learning_rate": 0.00014803241023588637,
1937
+ "entropy": 0.3992158595472574,
1938
+ "num_tokens": 15449277.0,
1939
+ "mean_token_accuracy": 0.8842400945723057,
1940
+ "epoch": 3.738308457711443,
1941
+ "step": 1880
1942
+ },
1943
+ {
1944
+ "loss": 0.3576947212219238,
1945
+ "grad_norm": 1.7177051305770874,
1946
+ "learning_rate": 0.00014745467815500157,
1947
+ "entropy": 0.37072331327944996,
1948
+ "num_tokens": 15529651.0,
1949
+ "mean_token_accuracy": 0.8907733589410782,
1950
+ "epoch": 3.7582089552238807,
1951
+ "step": 1890
1952
+ },
1953
+ {
1954
+ "loss": 0.3160592794418335,
1955
+ "grad_norm": 2.013315200805664,
1956
+ "learning_rate": 0.0001468748945108131,
1957
+ "entropy": 0.3132282990962267,
1958
+ "num_tokens": 15612895.0,
1959
+ "mean_token_accuracy": 0.8976165451109409,
1960
+ "epoch": 3.7781094527363184,
1961
+ "step": 1900
1962
+ },
1963
+ {
1964
+ "loss": 0.32902114391326903,
1965
+ "grad_norm": 1.553937554359436,
1966
+ "learning_rate": 0.00014629308436856,
1967
+ "entropy": 0.3400567851960659,
1968
+ "num_tokens": 15695353.0,
1969
+ "mean_token_accuracy": 0.8929221451282501,
1970
+ "epoch": 3.798009950248756,
1971
+ "step": 1910
1972
+ },
1973
+ {
1974
+ "loss": 0.38461942672729493,
1975
+ "grad_norm": 2.164433717727661,
1976
+ "learning_rate": 0.0001457092728810909,
1977
+ "entropy": 0.36988206636160614,
1978
+ "num_tokens": 15775948.0,
1979
+ "mean_token_accuracy": 0.8848864234983921,
1980
+ "epoch": 3.817910447761194,
1981
+ "step": 1920
1982
+ },
1983
+ {
1984
+ "loss": 0.36692650318145753,
1985
+ "grad_norm": 1.9492835998535156,
1986
+ "learning_rate": 0.00014512348528777675,
1987
+ "entropy": 0.38128890469670296,
1988
+ "num_tokens": 15859777.0,
1989
+ "mean_token_accuracy": 0.8892743974924088,
1990
+ "epoch": 3.837810945273632,
1991
+ "step": 1930
1992
+ },
1993
+ {
1994
+ "loss": 0.38012266159057617,
1995
+ "grad_norm": 2.2882587909698486,
1996
+ "learning_rate": 0.00014453574691341957,
1997
+ "entropy": 0.3851106442511082,
1998
+ "num_tokens": 15940237.0,
1999
+ "mean_token_accuracy": 0.8856042474508286,
2000
+ "epoch": 3.8577114427860697,
2001
+ "step": 1940
2002
+ },
2003
+ {
2004
+ "loss": 0.362756085395813,
2005
+ "grad_norm": 1.8618828058242798,
2006
+ "learning_rate": 0.00014394608316715764,
2007
+ "entropy": 0.3671020517125726,
2008
+ "num_tokens": 16024873.0,
2009
+ "mean_token_accuracy": 0.8824686802923679,
2010
+ "epoch": 3.8776119402985074,
2011
+ "step": 1950
2012
+ },
2013
+ {
2014
+ "loss": 0.3531129598617554,
2015
+ "grad_norm": 1.7716271877288818,
2016
+ "learning_rate": 0.00014335451954136712,
2017
+ "entropy": 0.36526877535507085,
2018
+ "num_tokens": 16107783.0,
2019
+ "mean_token_accuracy": 0.8884353943169117,
2020
+ "epoch": 3.897512437810945,
2021
+ "step": 1960
2022
+ },
2023
+ {
2024
+ "loss": 0.35922248363494874,
2025
+ "grad_norm": 1.609451174736023,
2026
+ "learning_rate": 0.00014276108161055977,
2027
+ "entropy": 0.3464937123470008,
2028
+ "num_tokens": 16188195.0,
2029
+ "mean_token_accuracy": 0.8906741172075272,
2030
+ "epoch": 3.917412935323383,
2031
+ "step": 1970
2032
+ },
2033
+ {
2034
+ "loss": 0.37455728054046633,
2035
+ "grad_norm": 1.922342300415039,
2036
+ "learning_rate": 0.00014216579503027748,
2037
+ "entropy": 0.3929541580379009,
2038
+ "num_tokens": 16270260.0,
2039
+ "mean_token_accuracy": 0.8852056123316288,
2040
+ "epoch": 3.937313432835821,
2041
+ "step": 1980
2042
+ },
2043
+ {
2044
+ "loss": 0.3485581636428833,
2045
+ "grad_norm": 2.2290749549865723,
2046
+ "learning_rate": 0.000141568685535983,
2047
+ "entropy": 0.3450996743515134,
2048
+ "num_tokens": 16350810.0,
2049
+ "mean_token_accuracy": 0.8908451579511165,
2050
+ "epoch": 3.9572139303482587,
2051
+ "step": 1990
2052
+ },
2053
+ {
2054
+ "loss": 0.3451216697692871,
2055
+ "grad_norm": 1.5832468271255493,
2056
+ "learning_rate": 0.00014096977894194741,
2057
+ "entropy": 0.3624888777732849,
2058
+ "num_tokens": 16436556.0,
2059
+ "mean_token_accuracy": 0.899410167336464,
2060
+ "epoch": 3.9771144278606965,
2061
+ "step": 2000
2062
+ },
2063
+ {
2064
+ "loss": 0.3591928243637085,
2065
+ "grad_norm": 2.0904476642608643,
2066
+ "learning_rate": 0.0001403691011401342,
2067
+ "entropy": 0.368430376984179,
2068
+ "num_tokens": 16518893.0,
2069
+ "mean_token_accuracy": 0.8849965989589691,
2070
+ "epoch": 3.997014925373134,
2071
+ "step": 2010
2072
+ },
2073
+ {
2074
+ "eval_loss": 1.1314672231674194,
2075
+ "eval_runtime": 88.5632,
2076
+ "eval_samples_per_second": 11.687,
2077
+ "eval_steps_per_second": 5.849,
2078
+ "eval_entropy": 0.47168861136818485,
2079
+ "eval_num_tokens": 16530624.0,
2080
+ "eval_mean_token_accuracy": 0.7530034936417944,
2081
+ "epoch": 4.0,
2082
+ "step": 2012
2083
+ },
2084
+ {
2085
+ "loss": 0.23610332012176513,
2086
+ "grad_norm": 1.5669670104980469,
2087
+ "learning_rate": 0.00013976667809907967,
2088
+ "entropy": 0.2812192935104433,
2089
+ "num_tokens": 16594663.0,
2090
+ "mean_token_accuracy": 0.9318282988510633,
2091
+ "epoch": 4.01592039800995,
2092
+ "step": 2020
2093
+ },
2094
+ {
2095
+ "loss": 0.19126780033111573,
2096
+ "grad_norm": 1.8125327825546265,
2097
+ "learning_rate": 0.00013916253586277046,
2098
+ "entropy": 0.2425563176162541,
2099
+ "num_tokens": 16677999.0,
2100
+ "mean_token_accuracy": 0.9397158071398735,
2101
+ "epoch": 4.035820895522388,
2102
+ "step": 2030
2103
+ },
2104
+ {
2105
+ "loss": 0.18233511447906495,
2106
+ "grad_norm": 1.2361959218978882,
2107
+ "learning_rate": 0.00013855670054951764,
2108
+ "entropy": 0.21818739883601665,
2109
+ "num_tokens": 16762226.0,
2110
+ "mean_token_accuracy": 0.9379066251218319,
2111
+ "epoch": 4.055721393034826,
2112
+ "step": 2040
2113
+ },
2114
+ {
2115
+ "loss": 0.1836015224456787,
2116
+ "grad_norm": 1.4206912517547607,
2117
+ "learning_rate": 0.00013794919835082733,
2118
+ "entropy": 0.21597656197845935,
2119
+ "num_tokens": 16843862.0,
2120
+ "mean_token_accuracy": 0.9375724367797375,
2121
+ "epoch": 4.075621890547263,
2122
+ "step": 2050
2123
+ },
2124
+ {
2125
+ "loss": 0.21675570011138917,
2126
+ "grad_norm": 1.6368014812469482,
2127
+ "learning_rate": 0.00013734005553026863,
2128
+ "entropy": 0.24125779159367083,
2129
+ "num_tokens": 16927499.0,
2130
+ "mean_token_accuracy": 0.9257908374071121,
2131
+ "epoch": 4.095522388059702,
2132
+ "step": 2060
2133
+ },
2134
+ {
2135
+ "loss": 0.2040395975112915,
2136
+ "grad_norm": 1.9235601425170898,
2137
+ "learning_rate": 0.00013672929842233807,
2138
+ "entropy": 0.24503452610224485,
2139
+ "num_tokens": 17009422.0,
2140
+ "mean_token_accuracy": 0.93132429048419,
2141
+ "epoch": 4.115422885572139,
2142
+ "step": 2070
2143
+ },
2144
+ {
2145
+ "loss": 0.21393344402313233,
2146
+ "grad_norm": 1.7291146516799927,
2147
+ "learning_rate": 0.00013611695343132118,
2148
+ "entropy": 0.23869293741881847,
2149
+ "num_tokens": 17091305.0,
2150
+ "mean_token_accuracy": 0.9325790904462338,
2151
+ "epoch": 4.135323383084577,
2152
+ "step": 2080
2153
+ },
2154
+ {
2155
+ "loss": 0.21835823059082032,
2156
+ "grad_norm": 1.7889643907546997,
2157
+ "learning_rate": 0.00013550304703015083,
2158
+ "entropy": 0.25416098097339274,
2159
+ "num_tokens": 17173267.0,
2160
+ "mean_token_accuracy": 0.9285577841103076,
2161
+ "epoch": 4.155223880597015,
2162
+ "step": 2090
2163
+ },
2164
+ {
2165
+ "loss": 0.21137917041778564,
2166
+ "grad_norm": 2.037461757659912,
2167
+ "learning_rate": 0.000134887605759263,
2168
+ "entropy": 0.24891419094055892,
2169
+ "num_tokens": 17256705.0,
2170
+ "mean_token_accuracy": 0.931586105376482,
2171
+ "epoch": 4.1751243781094525,
2172
+ "step": 2100
2173
+ },
2174
+ {
2175
+ "loss": 0.20982325077056885,
2176
+ "grad_norm": 1.7877676486968994,
2177
+ "learning_rate": 0.00013427065622544914,
2178
+ "entropy": 0.22691308157518505,
2179
+ "num_tokens": 17338188.0,
2180
+ "mean_token_accuracy": 0.9303626663982868,
2181
+ "epoch": 4.195024875621891,
2182
+ "step": 2110
2183
+ },
2184
+ {
2185
+ "loss": 0.22454733848571778,
2186
+ "grad_norm": 1.761404037475586,
2187
+ "learning_rate": 0.0001336522251007061,
2188
+ "entropy": 0.24377617733553053,
2189
+ "num_tokens": 17418864.0,
2190
+ "mean_token_accuracy": 0.925045907497406,
2191
+ "epoch": 4.214925373134328,
2192
+ "step": 2120
2193
+ },
2194
+ {
2195
+ "loss": 0.20816371440887452,
2196
+ "grad_norm": 1.6466680765151978,
2197
+ "learning_rate": 0.00013303233912108283,
2198
+ "entropy": 0.24038536921143533,
2199
+ "num_tokens": 17499343.0,
2200
+ "mean_token_accuracy": 0.93259956240654,
2201
+ "epoch": 4.234825870646766,
2202
+ "step": 2130
2203
+ },
2204
+ {
2205
+ "loss": 0.20254700183868407,
2206
+ "grad_norm": 1.6878200769424438,
2207
+ "learning_rate": 0.00013241102508552475,
2208
+ "entropy": 0.2293321386910975,
2209
+ "num_tokens": 17582158.0,
2210
+ "mean_token_accuracy": 0.9291081473231315,
2211
+ "epoch": 4.254726368159204,
2212
+ "step": 2140
2213
+ },
2214
+ {
2215
+ "loss": 0.21738946437835693,
2216
+ "grad_norm": 1.8285473585128784,
2217
+ "learning_rate": 0.00013178830985471505,
2218
+ "entropy": 0.23572409860789775,
2219
+ "num_tokens": 17662645.0,
2220
+ "mean_token_accuracy": 0.9273534499108791,
2221
+ "epoch": 4.2746268656716415,
2222
+ "step": 2150
2223
+ },
2224
+ {
2225
+ "loss": 0.22995378971099853,
2226
+ "grad_norm": 2.1973538398742676,
2227
+ "learning_rate": 0.00013116422034991347,
2228
+ "entropy": 0.2525733551941812,
2229
+ "num_tokens": 17743786.0,
2230
+ "mean_token_accuracy": 0.9257130898535252,
2231
+ "epoch": 4.29452736318408,
2232
+ "step": 2160
2233
+ },
2234
+ {
2235
+ "loss": 0.19108937978744506,
2236
+ "grad_norm": 1.4349671602249146,
2237
+ "learning_rate": 0.00013053878355179244,
2238
+ "entropy": 0.23723841486498715,
2239
+ "num_tokens": 17826549.0,
2240
+ "mean_token_accuracy": 0.9368034176528454,
2241
+ "epoch": 4.314427860696517,
2242
+ "step": 2170
2243
+ },
2244
+ {
2245
+ "loss": 0.21866753101348876,
2246
+ "grad_norm": 1.8653446435928345,
2247
+ "learning_rate": 0.00012991202649927056,
2248
+ "entropy": 0.2479944357648492,
2249
+ "num_tokens": 17910217.0,
2250
+ "mean_token_accuracy": 0.928318190574646,
2251
+ "epoch": 4.334328358208955,
2252
+ "step": 2180
2253
+ },
2254
+ {
2255
+ "loss": 0.21983544826507567,
2256
+ "grad_norm": 1.4839264154434204,
2257
+ "learning_rate": 0.00012928397628834395,
2258
+ "entropy": 0.2442836315371096,
2259
+ "num_tokens": 17993954.0,
2260
+ "mean_token_accuracy": 0.931411312520504,
2261
+ "epoch": 4.354228855721393,
2262
+ "step": 2190
2263
+ },
2264
+ {
2265
+ "loss": 0.21417014598846434,
2266
+ "grad_norm": 1.8984490633010864,
2267
+ "learning_rate": 0.0001286546600709144,
2268
+ "entropy": 0.26048750150948763,
2269
+ "num_tokens": 18074834.0,
2270
+ "mean_token_accuracy": 0.9260428458452225,
2271
+ "epoch": 4.374129353233831,
2272
+ "step": 2200
2273
+ },
2274
+ {
2275
+ "loss": 0.20807394981384278,
2276
+ "grad_norm": 1.6519417762756348,
2277
+ "learning_rate": 0.00012802410505361592,
2278
+ "entropy": 0.2315901905298233,
2279
+ "num_tokens": 18157020.0,
2280
+ "mean_token_accuracy": 0.9260378688573837,
2281
+ "epoch": 4.394029850746269,
2282
+ "step": 2210
2283
+ },
2284
+ {
2285
+ "loss": 0.20873630046844482,
2286
+ "grad_norm": 1.6588348150253296,
2287
+ "learning_rate": 0.0001273923384966383,
2288
+ "entropy": 0.22860154528170823,
2289
+ "num_tokens": 18239163.0,
2290
+ "mean_token_accuracy": 0.9287099935114383,
2291
+ "epoch": 4.413930348258706,
2292
+ "step": 2220
2293
+ },
2294
+ {
2295
+ "loss": 0.20987870693206787,
2296
+ "grad_norm": 1.6810976266860962,
2297
+ "learning_rate": 0.00012675938771254872,
2298
+ "entropy": 0.2467828250490129,
2299
+ "num_tokens": 18319296.0,
2300
+ "mean_token_accuracy": 0.9340152963995934,
2301
+ "epoch": 4.433830845771144,
2302
+ "step": 2230
2303
+ },
2304
+ {
2305
+ "loss": 0.2371424674987793,
2306
+ "grad_norm": 1.9242221117019653,
2307
+ "learning_rate": 0.00012612528006511093,
2308
+ "entropy": 0.23780915308743716,
2309
+ "num_tokens": 18400836.0,
2310
+ "mean_token_accuracy": 0.9244723625481128,
2311
+ "epoch": 4.453731343283582,
2312
+ "step": 2240
2313
+ },
2314
+ {
2315
+ "loss": 0.23316283226013185,
2316
+ "grad_norm": 1.9295995235443115,
2317
+ "learning_rate": 0.0001254900429681023,
2318
+ "entropy": 0.2639192405156791,
2319
+ "num_tokens": 18481842.0,
2320
+ "mean_token_accuracy": 0.9205155313014984,
2321
+ "epoch": 4.47363184079602,
2322
+ "step": 2250
2323
+ },
2324
+ {
2325
+ "loss": 0.23252336978912352,
2326
+ "grad_norm": 2.227384090423584,
2327
+ "learning_rate": 0.0001248537038841285,
2328
+ "entropy": 0.25302106477320196,
2329
+ "num_tokens": 18563047.0,
2330
+ "mean_token_accuracy": 0.9212457716464997,
2331
+ "epoch": 4.493532338308458,
2332
+ "step": 2260
2333
+ },
2334
+ {
2335
+ "loss": 0.21879236698150634,
2336
+ "grad_norm": 1.714643955230713,
2337
+ "learning_rate": 0.0001242162903234365,
2338
+ "entropy": 0.24494377207010984,
2339
+ "num_tokens": 18645551.0,
2340
+ "mean_token_accuracy": 0.9271452203392982,
2341
+ "epoch": 4.513432835820895,
2342
+ "step": 2270
2343
+ },
2344
+ {
2345
+ "loss": 0.2192246913909912,
2346
+ "grad_norm": 1.9092953205108643,
2347
+ "learning_rate": 0.00012357782984272504,
2348
+ "entropy": 0.23652592720463872,
2349
+ "num_tokens": 18727345.0,
2350
+ "mean_token_accuracy": 0.9352771908044815,
2351
+ "epoch": 4.533333333333333,
2352
+ "step": 2280
2353
+ },
2354
+ {
2355
+ "loss": 0.21921801567077637,
2356
+ "grad_norm": 1.6674190759658813,
2357
+ "learning_rate": 0.0001229383500439534,
2358
+ "entropy": 0.25822186917066575,
2359
+ "num_tokens": 18808521.0,
2360
+ "mean_token_accuracy": 0.926652018725872,
2361
+ "epoch": 4.553233830845771,
2362
+ "step": 2290
2363
+ },
2364
+ {
2365
+ "loss": 0.22446150779724122,
2366
+ "grad_norm": 1.7135406732559204,
2367
+ "learning_rate": 0.00012229787857314801,
2368
+ "entropy": 0.2487781412899494,
2369
+ "num_tokens": 18893033.0,
2370
+ "mean_token_accuracy": 0.9268894538283348,
2371
+ "epoch": 4.573134328358209,
2372
+ "step": 2300
2373
+ },
2374
+ {
2375
+ "loss": 0.2381376266479492,
2376
+ "grad_norm": 1.7335484027862549,
2377
+ "learning_rate": 0.00012165644311920741,
2378
+ "entropy": 0.24882857529446484,
2379
+ "num_tokens": 18975174.0,
2380
+ "mean_token_accuracy": 0.921249159425497,
2381
+ "epoch": 4.593034825870647,
2382
+ "step": 2310
2383
+ },
2384
+ {
2385
+ "loss": 0.24528043270111083,
2386
+ "grad_norm": 1.5288317203521729,
2387
+ "learning_rate": 0.000121014071412705,
2388
+ "entropy": 0.2677458773367107,
2389
+ "num_tokens": 19055831.0,
2390
+ "mean_token_accuracy": 0.9206675894558429,
2391
+ "epoch": 4.612935323383084,
2392
+ "step": 2320
2393
+ },
2394
+ {
2395
+ "loss": 0.23617784976959227,
2396
+ "grad_norm": 1.570332646369934,
2397
+ "learning_rate": 0.00012037079122469044,
2398
+ "entropy": 0.26810272233560684,
2399
+ "num_tokens": 19137919.0,
2400
+ "mean_token_accuracy": 0.9237006165087223,
2401
+ "epoch": 4.632835820895522,
2402
+ "step": 2330
2403
+ },
2404
+ {
2405
+ "loss": 0.2327519416809082,
2406
+ "grad_norm": 1.6146273612976074,
2407
+ "learning_rate": 0.00011972663036548884,
2408
+ "entropy": 0.25389791969209907,
2409
+ "num_tokens": 19219581.0,
2410
+ "mean_token_accuracy": 0.9215208649635315,
2411
+ "epoch": 4.65273631840796,
2412
+ "step": 2340
2413
+ },
2414
+ {
2415
+ "loss": 0.22447736263275148,
2416
+ "grad_norm": 1.8854310512542725,
2417
+ "learning_rate": 0.0001190816166834985,
2418
+ "entropy": 0.2654360429383814,
2419
+ "num_tokens": 19303929.0,
2420
+ "mean_token_accuracy": 0.9278781965374947,
2421
+ "epoch": 4.672636815920398,
2422
+ "step": 2350
2423
+ },
2424
+ {
2425
+ "loss": 0.21850359439849854,
2426
+ "grad_norm": 1.862807035446167,
2427
+ "learning_rate": 0.00011843577806398705,
2428
+ "entropy": 0.24387211743742226,
2429
+ "num_tokens": 19387796.0,
2430
+ "mean_token_accuracy": 0.9249358050525188,
2431
+ "epoch": 4.692537313432836,
2432
+ "step": 2360
2433
+ },
2434
+ {
2435
+ "loss": 0.21842756271362304,
2436
+ "grad_norm": 1.6853046417236328,
2437
+ "learning_rate": 0.00011778914242788588,
2438
+ "entropy": 0.2461325339972973,
2439
+ "num_tokens": 19469134.0,
2440
+ "mean_token_accuracy": 0.9314172364771366,
2441
+ "epoch": 4.712437810945273,
2442
+ "step": 2370
2443
+ },
2444
+ {
2445
+ "loss": 0.23487024307250975,
2446
+ "grad_norm": 1.6848011016845703,
2447
+ "learning_rate": 0.00011714173773058304,
2448
+ "entropy": 0.2532777524553239,
2449
+ "num_tokens": 19551990.0,
2450
+ "mean_token_accuracy": 0.9258120492100715,
2451
+ "epoch": 4.732338308457711,
2452
+ "step": 2380
2453
+ },
2454
+ {
2455
+ "loss": 0.2275081157684326,
2456
+ "grad_norm": 1.8661749362945557,
2457
+ "learning_rate": 0.00011649359196071459,
2458
+ "entropy": 0.24442209769040346,
2459
+ "num_tokens": 19632037.0,
2460
+ "mean_token_accuracy": 0.9279134057462215,
2461
+ "epoch": 4.7522388059701495,
2462
+ "step": 2390
2463
+ },
2464
+ {
2465
+ "loss": 0.23597207069396972,
2466
+ "grad_norm": 1.6075812578201294,
2467
+ "learning_rate": 0.00011584473313895483,
2468
+ "entropy": 0.2620496856048703,
2469
+ "num_tokens": 19715489.0,
2470
+ "mean_token_accuracy": 0.9233741469681263,
2471
+ "epoch": 4.772139303482587,
2472
+ "step": 2400
2473
+ },
2474
+ {
2475
+ "loss": 0.23563318252563475,
2476
+ "grad_norm": 1.8898063898086548,
2477
+ "learning_rate": 0.00011519518931680463,
2478
+ "entropy": 0.24913959829136728,
2479
+ "num_tokens": 19800527.0,
2480
+ "mean_token_accuracy": 0.9218057818710804,
2481
+ "epoch": 4.792039800995025,
2482
+ "step": 2410
2483
+ },
2484
+ {
2485
+ "loss": 0.23714659214019776,
2486
+ "grad_norm": 1.7826662063598633,
2487
+ "learning_rate": 0.00011454498857537893,
2488
+ "entropy": 0.2391042524948716,
2489
+ "num_tokens": 19883911.0,
2490
+ "mean_token_accuracy": 0.9197839744389057,
2491
+ "epoch": 4.811940298507462,
2492
+ "step": 2420
2493
+ },
2494
+ {
2495
+ "loss": 0.2159576416015625,
2496
+ "grad_norm": 1.67578125,
2497
+ "learning_rate": 0.00011389415902419251,
2498
+ "entropy": 0.25182965537533164,
2499
+ "num_tokens": 19968155.0,
2500
+ "mean_token_accuracy": 0.922177518159151,
2501
+ "epoch": 4.8318407960199,
2502
+ "step": 2430
2503
+ },
2504
+ {
2505
+ "loss": 0.22929255962371825,
2506
+ "grad_norm": 1.9571832418441772,
2507
+ "learning_rate": 0.00011324272879994509,
2508
+ "entropy": 0.26527754878625276,
2509
+ "num_tokens": 20049798.0,
2510
+ "mean_token_accuracy": 0.9268258795142174,
2511
+ "epoch": 4.8517412935323385,
2512
+ "step": 2440
2513
+ },
2514
+ {
2515
+ "loss": 0.2240236520767212,
2516
+ "grad_norm": 1.4346449375152588,
2517
+ "learning_rate": 0.00011259072606530452,
2518
+ "entropy": 0.23304696725681423,
2519
+ "num_tokens": 20131529.0,
2520
+ "mean_token_accuracy": 0.9269200436770916,
2521
+ "epoch": 4.871641791044776,
2522
+ "step": 2450
2523
+ },
2524
+ {
2525
+ "loss": 0.23434126377105713,
2526
+ "grad_norm": 1.7825595140457153,
2527
+ "learning_rate": 0.0001119381790076896,
2528
+ "entropy": 0.24932249234989284,
2529
+ "num_tokens": 20214183.0,
2530
+ "mean_token_accuracy": 0.9212065361440182,
2531
+ "epoch": 4.891542288557214,
2532
+ "step": 2460
2533
+ },
2534
+ {
2535
+ "loss": 0.24337990283966066,
2536
+ "grad_norm": 2.058513641357422,
2537
+ "learning_rate": 0.00011128511583805122,
2538
+ "entropy": 0.2578001916408539,
2539
+ "num_tokens": 20297522.0,
2540
+ "mean_token_accuracy": 0.9210958659648896,
2541
+ "epoch": 4.911442786069652,
2542
+ "step": 2470
2543
+ },
2544
+ {
2545
+ "loss": 0.23612210750579835,
2546
+ "grad_norm": 1.8045240640640259,
2547
+ "learning_rate": 0.00011063156478965293,
2548
+ "entropy": 0.26722495248541234,
2549
+ "num_tokens": 20382566.0,
2550
+ "mean_token_accuracy": 0.9243033178150654,
2551
+ "epoch": 4.931343283582089,
2552
+ "step": 2480
2553
+ },
2554
+ {
2555
+ "loss": 0.2361851453781128,
2556
+ "grad_norm": 2.051356792449951,
2557
+ "learning_rate": 0.00010997755411685022,
2558
+ "entropy": 0.2558716372586787,
2559
+ "num_tokens": 20463700.0,
2560
+ "mean_token_accuracy": 0.9235666677355766,
2561
+ "epoch": 4.951243781094528,
2562
+ "step": 2490
2563
+ },
2564
+ {
2565
+ "loss": 0.22444696426391603,
2566
+ "grad_norm": 1.8039820194244385,
2567
+ "learning_rate": 0.0001093231120938692,
2568
+ "entropy": 0.25451266625896096,
2569
+ "num_tokens": 20544398.0,
2570
+ "mean_token_accuracy": 0.9238814383745193,
2571
+ "epoch": 4.971144278606965,
2572
+ "step": 2500
2573
+ },
2574
+ {
2575
+ "loss": 0.22053093910217286,
2576
+ "grad_norm": 1.730843186378479,
2577
+ "learning_rate": 0.000108668267013584,
2578
+ "entropy": 0.23836419014260174,
2579
+ "num_tokens": 20627521.0,
2580
+ "mean_token_accuracy": 0.9234050408005714,
2581
+ "epoch": 4.991044776119403,
2582
+ "step": 2510
2583
+ },
2584
+ {
2585
+ "eval_loss": 1.2360295057296753,
2586
+ "eval_runtime": 88.484,
2587
+ "eval_samples_per_second": 11.697,
2588
+ "eval_steps_per_second": 5.854,
2589
+ "eval_entropy": 0.3996568917365148,
2590
+ "eval_num_tokens": 20663280.0,
2591
+ "eval_mean_token_accuracy": 0.7523371222856883,
2592
+ "epoch": 5.0,
2593
+ "step": 2515
2594
+ },
2595
+ {
2596
+ "loss": 0.17534947395324707,
2597
+ "grad_norm": 1.2422212362289429,
2598
+ "learning_rate": 0.0001080130471862938,
2599
+ "entropy": 0.22914624449453855,
2600
+ "num_tokens": 20704454.0,
2601
+ "mean_token_accuracy": 0.9460896575137189,
2602
+ "epoch": 5.009950248756219,
2603
+ "step": 2520
2604
+ },
2605
+ {
2606
+ "loss": 0.13762762546539306,
2607
+ "grad_norm": 1.7252610921859741,
2608
+ "learning_rate": 0.00010735748093849889,
2609
+ "entropy": 0.1832891060039401,
2610
+ "num_tokens": 20787381.0,
2611
+ "mean_token_accuracy": 0.950270488858223,
2612
+ "epoch": 5.029850746268656,
2613
+ "step": 2530
2614
+ },
2615
+ {
2616
+ "loss": 0.12815582752227783,
2617
+ "grad_norm": 1.141512393951416,
2618
+ "learning_rate": 0.000106701596611676,
2619
+ "entropy": 0.17471029767766594,
2620
+ "num_tokens": 20865490.0,
2621
+ "mean_token_accuracy": 0.9606716863811016,
2622
+ "epoch": 5.0497512437810945,
2623
+ "step": 2540
2624
+ },
2625
+ {
2626
+ "loss": 0.12197397947311402,
2627
+ "grad_norm": 1.4147164821624756,
2628
+ "learning_rate": 0.00010604542256105308,
2629
+ "entropy": 0.1596170690841973,
2630
+ "num_tokens": 20950468.0,
2631
+ "mean_token_accuracy": 0.9579751871526241,
2632
+ "epoch": 5.069651741293533,
2633
+ "step": 2550
2634
+ },
2635
+ {
2636
+ "loss": 0.12484253644943237,
2637
+ "grad_norm": 1.597422480583191,
2638
+ "learning_rate": 0.00010538898715438345,
2639
+ "entropy": 0.16889475984498858,
2640
+ "num_tokens": 21034447.0,
2641
+ "mean_token_accuracy": 0.9622776135802269,
2642
+ "epoch": 5.08955223880597,
2643
+ "step": 2560
2644
+ },
2645
+ {
2646
+ "loss": 0.13227113485336303,
2647
+ "grad_norm": 1.4799704551696777,
2648
+ "learning_rate": 0.00010473231877071933,
2649
+ "entropy": 0.18726657945662736,
2650
+ "num_tokens": 21116879.0,
2651
+ "mean_token_accuracy": 0.9566851153969764,
2652
+ "epoch": 5.109452736318408,
2653
+ "step": 2570
2654
+ },
2655
+ {
2656
+ "loss": 0.13044761419296264,
2657
+ "grad_norm": 1.5792561769485474,
2658
+ "learning_rate": 0.00010407544579918509,
2659
+ "entropy": 0.1735865803901106,
2660
+ "num_tokens": 21196772.0,
2661
+ "mean_token_accuracy": 0.9568023070693016,
2662
+ "epoch": 5.129353233830845,
2663
+ "step": 2580
2664
+ },
2665
+ {
2666
+ "loss": 0.13447166681289674,
2667
+ "grad_norm": 1.6878907680511475,
2668
+ "learning_rate": 0.00010341839663774986,
2669
+ "entropy": 0.1790808086283505,
2670
+ "num_tokens": 21279157.0,
2671
+ "mean_token_accuracy": 0.957117061316967,
2672
+ "epoch": 5.149253731343284,
2673
+ "step": 2590
2674
+ },
2675
+ {
2676
+ "loss": 0.14305230379104614,
2677
+ "grad_norm": 1.554625153541565,
2678
+ "learning_rate": 0.00010276119969199973,
2679
+ "entropy": 0.16902345772832633,
2680
+ "num_tokens": 21360919.0,
2681
+ "mean_token_accuracy": 0.9540403395891189,
2682
+ "epoch": 5.169154228855722,
2683
+ "step": 2600
2684
+ },
2685
+ {
2686
+ "loss": 0.14377222061157227,
2687
+ "grad_norm": 1.1937791109085083,
2688
+ "learning_rate": 0.00010210388337390986,
2689
+ "entropy": 0.17945182584226133,
2690
+ "num_tokens": 21442198.0,
2691
+ "mean_token_accuracy": 0.95441407635808,
2692
+ "epoch": 5.189054726368159,
2693
+ "step": 2610
2694
+ },
2695
+ {
2696
+ "loss": 0.14531824588775635,
2697
+ "grad_norm": 1.5391591787338257,
2698
+ "learning_rate": 0.00010144647610061617,
2699
+ "entropy": 0.16971054766327143,
2700
+ "num_tokens": 21524694.0,
2701
+ "mean_token_accuracy": 0.9517566092312336,
2702
+ "epoch": 5.208955223880597,
2703
+ "step": 2620
2704
+ },
2705
+ {
2706
+ "loss": 0.13437284231185914,
2707
+ "grad_norm": 1.4127544164657593,
2708
+ "learning_rate": 0.00010078900629318666,
2709
+ "entropy": 0.16742002535611392,
2710
+ "num_tokens": 21610173.0,
2711
+ "mean_token_accuracy": 0.9551968172192573,
2712
+ "epoch": 5.2288557213930345,
2713
+ "step": 2630
2714
+ },
2715
+ {
2716
+ "loss": 0.1439760446548462,
2717
+ "grad_norm": 1.5808171033859253,
2718
+ "learning_rate": 0.00010013150237539291,
2719
+ "entropy": 0.17231757938861847,
2720
+ "num_tokens": 21691601.0,
2721
+ "mean_token_accuracy": 0.9500082366168499,
2722
+ "epoch": 5.248756218905473,
2723
+ "step": 2640
2724
+ },
2725
+ {
2726
+ "loss": 0.14982556104660033,
2727
+ "grad_norm": 1.6159675121307373,
2728
+ "learning_rate": 9.947399277248114e-05,
2729
+ "entropy": 0.18437913516536356,
2730
+ "num_tokens": 21773640.0,
2731
+ "mean_token_accuracy": 0.9472062647342682,
2732
+ "epoch": 5.268656716417911,
2733
+ "step": 2650
2734
+ },
2735
+ {
2736
+ "loss": 0.1454100251197815,
2737
+ "grad_norm": 1.3811414241790771,
2738
+ "learning_rate": 9.881650590994321e-05,
2739
+ "entropy": 0.17077813004143536,
2740
+ "num_tokens": 21856383.0,
2741
+ "mean_token_accuracy": 0.9554009273648262,
2742
+ "epoch": 5.288557213930348,
2743
+ "step": 2660
2744
+ },
2745
+ {
2746
+ "loss": 0.1529370903968811,
2747
+ "grad_norm": 2.063105583190918,
2748
+ "learning_rate": 9.81590702122881e-05,
2749
+ "entropy": 0.1808861500583589,
2750
+ "num_tokens": 21938383.0,
2751
+ "mean_token_accuracy": 0.948763107508421,
2752
+ "epoch": 5.308457711442786,
2753
+ "step": 2670
2754
+ },
2755
+ {
2756
+ "loss": 0.13557761907577515,
2757
+ "grad_norm": 1.606972575187683,
2758
+ "learning_rate": 9.750171410181268e-05,
2759
+ "entropy": 0.17785537512972951,
2760
+ "num_tokens": 22022051.0,
2761
+ "mean_token_accuracy": 0.9500960379838943,
2762
+ "epoch": 5.3283582089552235,
2763
+ "step": 2680
2764
+ },
2765
+ {
2766
+ "loss": 0.14070883989334107,
2767
+ "grad_norm": 1.4112268686294556,
2768
+ "learning_rate": 9.684446599737313e-05,
2769
+ "entropy": 0.17111114114522935,
2770
+ "num_tokens": 22103736.0,
2771
+ "mean_token_accuracy": 0.9535823926329613,
2772
+ "epoch": 5.348258706467662,
2773
+ "step": 2690
2774
+ },
2775
+ {
2776
+ "loss": 0.15326887369155884,
2777
+ "grad_norm": 1.5268429517745972,
2778
+ "learning_rate": 9.618735431315628e-05,
2779
+ "entropy": 0.17168783275410532,
2780
+ "num_tokens": 22187778.0,
2781
+ "mean_token_accuracy": 0.9476280555129051,
2782
+ "epoch": 5.3681592039801,
2783
+ "step": 2700
2784
+ },
2785
+ {
2786
+ "loss": 0.14742431640625,
2787
+ "grad_norm": 1.8421076536178589,
2788
+ "learning_rate": 9.553040745745131e-05,
2789
+ "entropy": 0.1878282987046987,
2790
+ "num_tokens": 22267019.0,
2791
+ "mean_token_accuracy": 0.9506012089550495,
2792
+ "epoch": 5.388059701492537,
2793
+ "step": 2710
2794
+ },
2795
+ {
2796
+ "loss": 0.15141576528549194,
2797
+ "grad_norm": 1.8283631801605225,
2798
+ "learning_rate": 9.487365383142144e-05,
2799
+ "entropy": 0.16636274261400102,
2800
+ "num_tokens": 22347911.0,
2801
+ "mean_token_accuracy": 0.952550733089447,
2802
+ "epoch": 5.407960199004975,
2803
+ "step": 2720
2804
+ },
2805
+ {
2806
+ "loss": 0.14472270011901855,
2807
+ "grad_norm": 1.6224932670593262,
2808
+ "learning_rate": 9.42171218278762e-05,
2809
+ "entropy": 0.18225679048337043,
2810
+ "num_tokens": 22429956.0,
2811
+ "mean_token_accuracy": 0.9502832368016243,
2812
+ "epoch": 5.4278606965174125,
2813
+ "step": 2730
2814
+ },
2815
+ {
2816
+ "loss": 0.1553632140159607,
2817
+ "grad_norm": 1.2014065980911255,
2818
+ "learning_rate": 9.3560839830044e-05,
2819
+ "entropy": 0.1860564828850329,
2820
+ "num_tokens": 22514805.0,
2821
+ "mean_token_accuracy": 0.9482575446367264,
2822
+ "epoch": 5.447761194029851,
2823
+ "step": 2740
2824
+ },
2825
+ {
2826
+ "loss": 0.1534173846244812,
2827
+ "grad_norm": 1.1483746767044067,
2828
+ "learning_rate": 9.290483621034485e-05,
2829
+ "entropy": 0.19335241308435797,
2830
+ "num_tokens": 22594162.0,
2831
+ "mean_token_accuracy": 0.9506968580186367,
2832
+ "epoch": 5.467661691542289,
2833
+ "step": 2750
2834
+ },
2835
+ {
2836
+ "loss": 0.1423805832862854,
2837
+ "grad_norm": 1.1802810430526733,
2838
+ "learning_rate": 9.224913932916401e-05,
2839
+ "entropy": 0.17522130645811557,
2840
+ "num_tokens": 22675103.0,
2841
+ "mean_token_accuracy": 0.9564061060547828,
2842
+ "epoch": 5.487562189054726,
2843
+ "step": 2760
2844
+ },
2845
+ {
2846
+ "loss": 0.16999696493148803,
2847
+ "grad_norm": 1.7545998096466064,
2848
+ "learning_rate": 9.159377753362577e-05,
2849
+ "entropy": 0.177274182299152,
2850
+ "num_tokens": 22756462.0,
2851
+ "mean_token_accuracy": 0.9441149212419987,
2852
+ "epoch": 5.507462686567164,
2853
+ "step": 2770
2854
+ },
2855
+ {
2856
+ "loss": 0.15599164962768555,
2857
+ "grad_norm": 1.9323716163635254,
2858
+ "learning_rate": 9.0938779156368e-05,
2859
+ "entropy": 0.19683469235897064,
2860
+ "num_tokens": 22837647.0,
2861
+ "mean_token_accuracy": 0.9493952229619026,
2862
+ "epoch": 5.5273631840796025,
2863
+ "step": 2780
2864
+ },
2865
+ {
2866
+ "loss": 0.1470987915992737,
2867
+ "grad_norm": 1.420862078666687,
2868
+ "learning_rate": 9.028417251431717e-05,
2869
+ "entropy": 0.19523371988907456,
2870
+ "num_tokens": 22918963.0,
2871
+ "mean_token_accuracy": 0.9484399504959583,
2872
+ "epoch": 5.54726368159204,
2873
+ "step": 2790
2874
+ },
2875
+ {
2876
+ "loss": 0.14478300809860228,
2877
+ "grad_norm": 1.446480393409729,
2878
+ "learning_rate": 8.962998590746434e-05,
2879
+ "entropy": 0.185859234072268,
2880
+ "num_tokens": 23004621.0,
2881
+ "mean_token_accuracy": 0.9514032013714313,
2882
+ "epoch": 5.567164179104478,
2883
+ "step": 2800
2884
+ },
2885
+ {
2886
+ "loss": 0.1497580051422119,
2887
+ "grad_norm": 1.7207252979278564,
2888
+ "learning_rate": 8.897624761764152e-05,
2889
+ "entropy": 0.17532276483252646,
2890
+ "num_tokens": 23088455.0,
2891
+ "mean_token_accuracy": 0.950810880959034,
2892
+ "epoch": 5.587064676616915,
2893
+ "step": 2810
2894
+ },
2895
+ {
2896
+ "loss": 0.13927290439605713,
2897
+ "grad_norm": 1.8666800260543823,
2898
+ "learning_rate": 8.832298590729904e-05,
2899
+ "entropy": 0.16813257774338125,
2900
+ "num_tokens": 23169585.0,
2901
+ "mean_token_accuracy": 0.9538753777742386,
2902
+ "epoch": 5.606965174129353,
2903
+ "step": 2820
2904
+ },
2905
+ {
2906
+ "loss": 0.15171360969543457,
2907
+ "grad_norm": 1.593052625656128,
2908
+ "learning_rate": 8.76702290182838e-05,
2909
+ "entropy": 0.17214933927170933,
2910
+ "num_tokens": 23252281.0,
2911
+ "mean_token_accuracy": 0.9514808319509029,
2912
+ "epoch": 5.6268656716417915,
2913
+ "step": 2830
2914
+ },
2915
+ {
2916
+ "loss": 0.1722355604171753,
2917
+ "grad_norm": 1.8211098909378052,
2918
+ "learning_rate": 8.701800517061814e-05,
2919
+ "entropy": 0.18063214914873243,
2920
+ "num_tokens": 23334920.0,
2921
+ "mean_token_accuracy": 0.9428443647921085,
2922
+ "epoch": 5.646766169154229,
2923
+ "step": 2840
2924
+ },
2925
+ {
2926
+ "loss": 0.15239322185516357,
2927
+ "grad_norm": 1.7044843435287476,
2928
+ "learning_rate": 8.636634256127995e-05,
2929
+ "entropy": 0.18335627112537622,
2930
+ "num_tokens": 23413119.0,
2931
+ "mean_token_accuracy": 0.9484156593680382,
2932
+ "epoch": 5.666666666666667,
2933
+ "step": 2850
2934
+ },
2935
+ {
2936
+ "loss": 0.15082073211669922,
2937
+ "grad_norm": 1.735919713973999,
2938
+ "learning_rate": 8.571526936298372e-05,
2939
+ "entropy": 0.18242506040260195,
2940
+ "num_tokens": 23496782.0,
2941
+ "mean_token_accuracy": 0.949397337436676,
2942
+ "epoch": 5.686567164179104,
2943
+ "step": 2860
2944
+ },
2945
+ {
2946
+ "loss": 0.14254711866378783,
2947
+ "grad_norm": 1.5481796264648438,
2948
+ "learning_rate": 8.506481372296233e-05,
2949
+ "entropy": 0.1668767651543021,
2950
+ "num_tokens": 23579345.0,
2951
+ "mean_token_accuracy": 0.9547528505325318,
2952
+ "epoch": 5.706467661691542,
2953
+ "step": 2870
2954
+ },
2955
+ {
2956
+ "loss": 0.16232469081878662,
2957
+ "grad_norm": 1.276487112045288,
2958
+ "learning_rate": 8.441500376175055e-05,
2959
+ "entropy": 0.17980635408312082,
2960
+ "num_tokens": 23661589.0,
2961
+ "mean_token_accuracy": 0.9454159937798977,
2962
+ "epoch": 5.726368159203981,
2963
+ "step": 2880
2964
+ },
2965
+ {
2966
+ "loss": 0.15084612369537354,
2967
+ "grad_norm": 1.5978649854660034,
2968
+ "learning_rate": 8.376586757196893e-05,
2969
+ "entropy": 0.17503846548497676,
2970
+ "num_tokens": 23745040.0,
2971
+ "mean_token_accuracy": 0.9508810967206955,
2972
+ "epoch": 5.746268656716418,
2973
+ "step": 2890
2974
+ },
2975
+ {
2976
+ "loss": 0.15577337741851807,
2977
+ "grad_norm": 1.5376180410385132,
2978
+ "learning_rate": 8.311743321710962e-05,
2979
+ "entropy": 0.17748838970437647,
2980
+ "num_tokens": 23828741.0,
2981
+ "mean_token_accuracy": 0.947283898293972,
2982
+ "epoch": 5.766169154228856,
2983
+ "step": 2900
2984
+ },
2985
+ {
2986
+ "loss": 0.16305129528045653,
2987
+ "grad_norm": 1.410995364189148,
2988
+ "learning_rate": 8.246972873032292e-05,
2989
+ "entropy": 0.19196225269697606,
2990
+ "num_tokens": 23909202.0,
2991
+ "mean_token_accuracy": 0.9454095579683781,
2992
+ "epoch": 5.786069651741293,
2993
+ "step": 2910
2994
+ },
2995
+ {
2996
+ "loss": 0.15421292781829835,
2997
+ "grad_norm": 1.559767723083496,
2998
+ "learning_rate": 8.182278211320556e-05,
2999
+ "entropy": 0.18598383609205485,
3000
+ "num_tokens": 23991867.0,
3001
+ "mean_token_accuracy": 0.9466542080044746,
3002
+ "epoch": 5.8059701492537314,
3003
+ "step": 2920
3004
+ },
3005
+ {
3006
+ "loss": 0.14998261928558348,
3007
+ "grad_norm": 1.3756979703903198,
3008
+ "learning_rate": 8.117662133458984e-05,
3009
+ "entropy": 0.18076377538964153,
3010
+ "num_tokens": 24073174.0,
3011
+ "mean_token_accuracy": 0.951042790710926,
3012
+ "epoch": 5.82587064676617,
3013
+ "step": 2930
3014
+ },
3015
+ {
3016
+ "loss": 0.133389675617218,
3017
+ "grad_norm": 1.2057956457138062,
3018
+ "learning_rate": 8.053127432933475e-05,
3019
+ "entropy": 0.17494694823399187,
3020
+ "num_tokens": 24157497.0,
3021
+ "mean_token_accuracy": 0.952403973788023,
3022
+ "epoch": 5.845771144278607,
3023
+ "step": 2940
3024
+ },
3025
+ {
3026
+ "loss": 0.15985946655273436,
3027
+ "grad_norm": 1.7077171802520752,
3028
+ "learning_rate": 7.98867689971182e-05,
3029
+ "entropy": 0.18435341790318488,
3030
+ "num_tokens": 24240308.0,
3031
+ "mean_token_accuracy": 0.9459442816674709,
3032
+ "epoch": 5.865671641791045,
3033
+ "step": 2950
3034
+ },
3035
+ {
3036
+ "loss": 0.1396994948387146,
3037
+ "grad_norm": 1.349231243133545,
3038
+ "learning_rate": 7.924313320123075e-05,
3039
+ "entropy": 0.16967748273164035,
3040
+ "num_tokens": 24321979.0,
3041
+ "mean_token_accuracy": 0.9525260709226131,
3042
+ "epoch": 5.885572139303482,
3043
+ "step": 2960
3044
+ },
3045
+ {
3046
+ "loss": 0.15299395322799683,
3047
+ "grad_norm": 1.283353328704834,
3048
+ "learning_rate": 7.860039476737118e-05,
3049
+ "entropy": 0.18029331043362617,
3050
+ "num_tokens": 24406277.0,
3051
+ "mean_token_accuracy": 0.9431722618639469,
3052
+ "epoch": 5.9054726368159205,
3053
+ "step": 2970
3054
+ },
3055
+ {
3056
+ "loss": 0.14895654916763307,
3057
+ "grad_norm": 1.2930275201797485,
3058
+ "learning_rate": 7.795858148244348e-05,
3059
+ "entropy": 0.18708901344798506,
3060
+ "num_tokens": 24489586.0,
3061
+ "mean_token_accuracy": 0.945338387042284,
3062
+ "epoch": 5.925373134328359,
3063
+ "step": 2980
3064
+ },
3065
+ {
3066
+ "loss": 0.16557281017303466,
3067
+ "grad_norm": 1.5504826307296753,
3068
+ "learning_rate": 7.731772109335558e-05,
3069
+ "entropy": 0.19565808903425932,
3070
+ "num_tokens": 24570771.0,
3071
+ "mean_token_accuracy": 0.9432554632425308,
3072
+ "epoch": 5.945273631840796,
3073
+ "step": 2990
3074
+ },
3075
+ {
3076
+ "loss": 0.1594693899154663,
3077
+ "grad_norm": 2.1034810543060303,
3078
+ "learning_rate": 7.667784130581963e-05,
3079
+ "entropy": 0.1870686740614474,
3080
+ "num_tokens": 24653405.0,
3081
+ "mean_token_accuracy": 0.9398707859218121,
3082
+ "epoch": 5.965174129353234,
3083
+ "step": 3000
3084
+ },
3085
+ {
3086
+ "loss": 0.1598202109336853,
3087
+ "grad_norm": 1.5232559442520142,
3088
+ "learning_rate": 7.603896978315446e-05,
3089
+ "entropy": 0.1835718069691211,
3090
+ "num_tokens": 24733064.0,
3091
+ "mean_token_accuracy": 0.9420231267809868,
3092
+ "epoch": 5.985074626865671,
3093
+ "step": 3010
3094
+ },
3095
+ {
3096
+ "eval_loss": 1.2987958192825317,
3097
+ "eval_runtime": 88.7372,
3098
+ "eval_samples_per_second": 11.664,
3099
+ "eval_steps_per_second": 5.837,
3100
+ "eval_entropy": 0.36483325235344266,
3101
+ "eval_num_tokens": 24795936.0,
3102
+ "eval_mean_token_accuracy": 0.7524902858559229,
3103
+ "epoch": 6.0,
3104
+ "step": 3018
3105
+ },
3106
+ {
3107
+ "train_runtime": 13547.859,
3108
+ "train_samples_per_second": 5.935,
3109
+ "train_steps_per_second": 0.371,
3110
+ "total_flos": 5.281255645485466e+16,
3111
+ "train_loss": 0.5501504028135771,
3112
+ "epoch": 6.0,
3113
+ "step": 3018
3114
+ },
3115
+ {
3116
+ "eval_loss": 1.013381004333496,
3117
+ "eval_runtime": 88.0872,
3118
+ "eval_samples_per_second": 11.75,
3119
+ "eval_steps_per_second": 5.881,
3120
+ "eval_entropy": 0.6918873670421052,
3121
+ "eval_num_tokens": 24795936.0,
3122
+ "eval_mean_token_accuracy": 0.7516806636537824,
3123
+ "epoch": 6.0,
3124
+ "step": 3018
3125
+ }
3126
+ ]
3127
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<|endoftext|>",
5
+ "eos_token": "<|endoftext|>",
6
+ "is_local": true,
7
+ "model_max_length": 2048,
8
+ "pad_token": "<|endoftext|>",
9
+ "tokenizer_class": "TokenizersBackend",
10
+ "unk_token": "<|endoftext|>"
11
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a46feeb2eab16954d3fefe8cda13490c466c544ca6063b8067a5806d5b663b51
3
+ size 5841