Hugging Face's logo

codegenstudio
/
codegen-350M-sql2nosql-lora

Text Generation
PEFT
Safetensors
lora
sql2nosql
Model card Files
xet
 Community
bhavanikambhampati commited on
Commit
b31ec90
·
verified ·
1 Parent(s): daf4001

Publish sql2nosql adapter (v3)

Browse files
Files changed (17) hide show
  1. README.md +8 -10
  2. adapter_config.json +45 -43
  3. adapter_model.safetensors +2 -2
  4. checkpoint-1500/README.md +209 -0
  5. checkpoint-1500/adapter_config.json +45 -0
  6. checkpoint-1500/adapter_model.safetensors +3 -0
  7. checkpoint-1500/optimizer.pt +3 -0
  8. checkpoint-1500/rng_state.pth +3 -0
  9. checkpoint-1500/scheduler.pt +3 -0
  10. checkpoint-1500/tokenizer.json +0 -0
  11. checkpoint-1500/tokenizer_config.json +11 -0
  12. checkpoint-1500/trainer_state.json +1576 -0
  13. checkpoint-1500/training_args.bin +3 -0
  14. run_metadata.json +3119 -0
  15. tokenizer.json +0 -0
  16. tokenizer_config.json +11 -0
  17. training_args.bin +3 -0
README.md CHANGED
@@ -1,20 +1,18 @@
1
  ---
2
- library_name: peft
3
  base_model: Salesforce/codegen-350M-multi
4
- tags:
5
- - lora
6
- - peft
7
- - code
8
- - sql2nosql
9
- - codegen
10
  pipeline_tag: text-generation
 
 
 
 
11
  ---
12
 
13
  # codegenstudio/codegen-350M-sql2nosql-lora
14
 
15
- LoRA adapter for **sql2nosql** on [Salesforce/codegen-350M-multi](https://huggingface.co/Salesforce/codegen-350M-multi).
16
 
17
- - Checkpoint version: `v2`
18
  - Task: `sql2nosql`
19
 
20
  ## Usage
@@ -31,4 +29,4 @@ model = AutoModelForCausalLM.from_pretrained(base)
31
  model = PeftModel.from_pretrained(model, adapter)
32
  ```
33
 
34
- Or use the multi-adapter API in this project's `hf-deploy/` package.
 
1
  ---
 
2
  base_model: Salesforce/codegen-350M-multi
3
+ library_name: peft
 
 
 
 
 
4
  pipeline_tag: text-generation
5
+ tags:
6
+ - lora
7
+ - peft
8
+ - sql2nosql
9
  ---
10
 
11
  # codegenstudio/codegen-350M-sql2nosql-lora
12
 
13
+ LoRA adapter for sql2nosql on Salesforce/codegen-350M-multi.
14
 
15
+ - Checkpoint version: `v3`
16
  - Task: `sql2nosql`
17
 
18
  ## Usage
 
29
  model = PeftModel.from_pretrained(model, adapter)
30
  ```
31
 
32
+ Or use the multi-adapter API in this project's `fastapi-deploy/` package.
adapter_config.json CHANGED
@@ -1,43 +1,45 @@
1
- {
2
- "alora_invocation_tokens": null,
3
- "alpha_pattern": {},
4
- "arrow_config": null,
5
- "auto_mapping": null,
6
- "base_model_name_or_path": "Salesforce/codegen-350M-multi",
7
- "bias": "none",
8
- "corda_config": null,
9
- "ensure_weight_tying": false,
10
- "eva_config": null,
11
- "exclude_modules": null,
12
- "fan_in_fan_out": false,
13
- "inference_mode": true,
14
- "init_lora_weights": true,
15
- "layer_replication": null,
16
- "layers_pattern": null,
17
- "layers_to_transform": null,
18
- "loftq_config": {},
19
- "lora_alpha": 32,
20
- "lora_bias": false,
21
- "lora_dropout": 0.05,
22
- "lora_ga_config": null,
23
- "megatron_config": null,
24
- "megatron_core": "megatron.core",
25
- "modules_to_save": null,
26
- "peft_type": "LORA",
27
- "peft_version": "0.19.1",
28
- "qalora_group_size": 16,
29
- "r": 16,
30
- "rank_pattern": {},
31
- "revision": null,
32
- "target_modules": [
33
- "qkv_proj",
34
- "out_proj"
35
- ],
36
- "target_parameters": null,
37
- "task_type": "CAUSAL_LM",
38
- "trainable_token_indices": null,
39
- "use_bdlora": null,
40
- "use_dora": false,
41
- "use_qalora": false,
42
- "use_rslora": false
43
- }
 
 
 
1
+ {
2
+ "alora_invocation_tokens": null,
3
+ "alpha_pattern": {},
4
+ "arrow_config": null,
5
+ "auto_mapping": null,
6
+ "base_model_name_or_path": "/kaggle/working/models/base/Salesforce__codegen-350M-multi",
7
+ "bias": "none",
8
+ "corda_config": null,
9
+ "ensure_weight_tying": false,
10
+ "eva_config": null,
11
+ "exclude_modules": null,
12
+ "fan_in_fan_out": false,
13
+ "inference_mode": true,
14
+ "init_lora_weights": true,
15
+ "layer_replication": null,
16
+ "layers_pattern": null,
17
+ "layers_to_transform": null,
18
+ "loftq_config": {},
19
+ "lora_alpha": 64,
20
+ "lora_bias": false,
21
+ "lora_dropout": 0.05,
22
+ "lora_ga_config": null,
23
+ "megatron_config": null,
24
+ "megatron_core": "megatron.core",
25
+ "modules_to_save": null,
26
+ "peft_type": "LORA",
27
+ "peft_version": "0.19.1",
28
+ "qalora_group_size": 16,
29
+ "r": 32,
30
+ "rank_pattern": {},
31
+ "revision": null,
32
+ "target_modules": [
33
+ "qkv_proj",
34
+ "out_proj",
35
+ "fc_in",
36
+ "fc_out"
37
+ ],
38
+ "target_parameters": null,
39
+ "task_type": "CAUSAL_LM",
40
+ "trainable_token_indices": null,
41
+ "use_bdlora": null,
42
+ "use_dora": false,
43
+ "use_qalora": false,
44
+ "use_rslora": false
45
+ }
adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:9a9adff81776fea7a3c3d173743cdd95bc64c835efd91bd2defa2904d6b8baad
3
- size 7874776
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d86d2e16c20d51d0831bcd0922ad392d93ebf6eaf67a8d77a04d23a511f9f4ee
3
+ size 41963912
checkpoint-1500/README.md ADDED
@@ -0,0 +1,209 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: /kaggle/working/models/base/Salesforce__codegen-350M-multi
3
+ library_name: peft
4
+ pipeline_tag: text-generation
5
+ tags:
6
+ - base_model:adapter:/kaggle/working/models/base/Salesforce__codegen-350M-multi
7
+ - lora
8
+ - sft
9
+ - transformers
10
+ - trl
11
+ ---
12
+
13
+ # Model Card for Model ID
14
+
15
+ <!-- Provide a quick summary of what the model is/does. -->
16
+
17
+
18
+
19
+ ## Model Details
20
+
21
+ ### Model Description
22
+
23
+ <!-- Provide a longer summary of what this model is. -->
24
+
25
+
26
+
27
+ - **Developed by:** [More Information Needed]
28
+ - **Funded by [optional]:** [More Information Needed]
29
+ - **Shared by [optional]:** [More Information Needed]
30
+ - **Model type:** [More Information Needed]
31
+ - **Language(s) (NLP):** [More Information Needed]
32
+ - **License:** [More Information Needed]
33
+ - **Finetuned from model [optional]:** [More Information Needed]
34
+
35
+ ### Model Sources [optional]
36
+
37
+ <!-- Provide the basic links for the model. -->
38
+
39
+ - **Repository:** [More Information Needed]
40
+ - **Paper [optional]:** [More Information Needed]
41
+ - **Demo [optional]:** [More Information Needed]
42
+
43
+ ## Uses
44
+
45
+ <!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
46
+
47
+ ### Direct Use
48
+
49
+ <!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
50
+
51
+ [More Information Needed]
52
+
53
+ ### Downstream Use [optional]
54
+
55
+ <!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
56
+
57
+ [More Information Needed]
58
+
59
+ ### Out-of-Scope Use
60
+
61
+ <!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
62
+
63
+ [More Information Needed]
64
+
65
+ ## Bias, Risks, and Limitations
66
+
67
+ <!-- This section is meant to convey both technical and sociotechnical limitations. -->
68
+
69
+ [More Information Needed]
70
+
71
+ ### Recommendations
72
+
73
+ <!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
74
+
75
+ Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
76
+
77
+ ## How to Get Started with the Model
78
+
79
+ Use the code below to get started with the model.
80
+
81
+ [More Information Needed]
82
+
83
+ ## Training Details
84
+
85
+ ### Training Data
86
+
87
+ <!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
88
+
89
+ [More Information Needed]
90
+
91
+ ### Training Procedure
92
+
93
+ <!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
94
+
95
+ #### Preprocessing [optional]
96
+
97
+ [More Information Needed]
98
+
99
+
100
+ #### Training Hyperparameters
101
+
102
+ - **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
103
+
104
+ #### Speeds, Sizes, Times [optional]
105
+
106
+ <!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
107
+
108
+ [More Information Needed]
109
+
110
+ ## Evaluation
111
+
112
+ <!-- This section describes the evaluation protocols and provides the results. -->
113
+
114
+ ### Testing Data, Factors & Metrics
115
+
116
+ #### Testing Data
117
+
118
+ <!-- This should link to a Dataset Card if possible. -->
119
+
120
+ [More Information Needed]
121
+
122
+ #### Factors
123
+
124
+ <!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
125
+
126
+ [More Information Needed]
127
+
128
+ #### Metrics
129
+
130
+ <!-- These are the evaluation metrics being used, ideally with a description of why. -->
131
+
132
+ [More Information Needed]
133
+
134
+ ### Results
135
+
136
+ [More Information Needed]
137
+
138
+ #### Summary
139
+
140
+
141
+
142
+ ## Model Examination [optional]
143
+
144
+ <!-- Relevant interpretability work for the model goes here -->
145
+
146
+ [More Information Needed]
147
+
148
+ ## Environmental Impact
149
+
150
+ <!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
151
+
152
+ Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
153
+
154
+ - **Hardware Type:** [More Information Needed]
155
+ - **Hours used:** [More Information Needed]
156
+ - **Cloud Provider:** [More Information Needed]
157
+ - **Compute Region:** [More Information Needed]
158
+ - **Carbon Emitted:** [More Information Needed]
159
+
160
+ ## Technical Specifications [optional]
161
+
162
+ ### Model Architecture and Objective
163
+
164
+ [More Information Needed]
165
+
166
+ ### Compute Infrastructure
167
+
168
+ [More Information Needed]
169
+
170
+ #### Hardware
171
+
172
+ [More Information Needed]
173
+
174
+ #### Software
175
+
176
+ [More Information Needed]
177
+
178
+ ## Citation [optional]
179
+
180
+ <!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
181
+
182
+ **BibTeX:**
183
+
184
+ [More Information Needed]
185
+
186
+ **APA:**
187
+
188
+ [More Information Needed]
189
+
190
+ ## Glossary [optional]
191
+
192
+ <!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
193
+
194
+ [More Information Needed]
195
+
196
+ ## More Information [optional]
197
+
198
+ [More Information Needed]
199
+
200
+ ## Model Card Authors [optional]
201
+
202
+ [More Information Needed]
203
+
204
+ ## Model Card Contact
205
+
206
+ [More Information Needed]
207
+ ### Framework versions
208
+
209
+ - PEFT 0.19.1
checkpoint-1500/adapter_config.json ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alora_invocation_tokens": null,
3
+ "alpha_pattern": {},
4
+ "arrow_config": null,
5
+ "auto_mapping": null,
6
+ "base_model_name_or_path": "/kaggle/working/models/base/Salesforce__codegen-350M-multi",
7
+ "bias": "none",
8
+ "corda_config": null,
9
+ "ensure_weight_tying": false,
10
+ "eva_config": null,
11
+ "exclude_modules": null,
12
+ "fan_in_fan_out": false,
13
+ "inference_mode": true,
14
+ "init_lora_weights": true,
15
+ "layer_replication": null,
16
+ "layers_pattern": null,
17
+ "layers_to_transform": null,
18
+ "loftq_config": {},
19
+ "lora_alpha": 64,
20
+ "lora_bias": false,
21
+ "lora_dropout": 0.05,
22
+ "lora_ga_config": null,
23
+ "megatron_config": null,
24
+ "megatron_core": "megatron.core",
25
+ "modules_to_save": null,
26
+ "peft_type": "LORA",
27
+ "peft_version": "0.19.1",
28
+ "qalora_group_size": 16,
29
+ "r": 32,
30
+ "rank_pattern": {},
31
+ "revision": null,
32
+ "target_modules": [
33
+ "qkv_proj",
34
+ "out_proj",
35
+ "fc_in",
36
+ "fc_out"
37
+ ],
38
+ "target_parameters": null,
39
+ "task_type": "CAUSAL_LM",
40
+ "trainable_token_indices": null,
41
+ "use_bdlora": null,
42
+ "use_dora": false,
43
+ "use_qalora": false,
44
+ "use_rslora": false
45
+ }
checkpoint-1500/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d86d2e16c20d51d0831bcd0922ad392d93ebf6eaf67a8d77a04d23a511f9f4ee
3
+ size 41963912
checkpoint-1500/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:184d01e8e2421af46f8a4147882138bd3b8afc4df0c4c58140c17c03fc56ac3c
3
+ size 84023435
checkpoint-1500/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3582adb3a9f101ad6035a563ab1d125e55caf4537ad7d48b018cc1649592f40a
3
+ size 14645
checkpoint-1500/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7ea5af5df7a88767be7b653584ef373a48b667022ff2327b8116f4ae737e77f1
3
+ size 1465
checkpoint-1500/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
checkpoint-1500/tokenizer_config.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<|endoftext|>",
5
+ "eos_token": "<|endoftext|>",
6
+ "is_local": true,
7
+ "model_max_length": 2048,
8
+ "pad_token": "<|endoftext|>",
9
+ "tokenizer_class": "TokenizersBackend",
10
+ "unk_token": "<|endoftext|>"
11
+ }
checkpoint-1500/trainer_state.json ADDED
@@ -0,0 +1,1576 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": 1500,
3
+ "best_metric": 0.03645886108279228,
4
+ "best_model_checkpoint": "/kaggle/working/models/checkpoints/v4/sql2nosql/checkpoint-1500",
5
+ "epoch": 3.0,
6
+ "eval_steps": 500,
7
+ "global_step": 1500,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "entropy": 1.1209821447730064,
14
+ "epoch": 0.020005001250312578,
15
+ "grad_norm": 1.7302765846252441,
16
+ "learning_rate": 7.2e-06,
17
+ "loss": 1.3475940704345704,
18
+ "mean_token_accuracy": 0.6775479473173618,
19
+ "num_tokens": 65756.0,
20
+ "step": 10
21
+ },
22
+ {
23
+ "entropy": 1.0718491852283478,
24
+ "epoch": 0.040010002500625155,
25
+ "grad_norm": 1.598680853843689,
26
+ "learning_rate": 1.52e-05,
27
+ "loss": 1.1782958984375,
28
+ "mean_token_accuracy": 0.6978455670177937,
29
+ "num_tokens": 127350.0,
30
+ "step": 20
31
+ },
32
+ {
33
+ "entropy": 0.9551009342074395,
34
+ "epoch": 0.06001500375093773,
35
+ "grad_norm": 1.7650020122528076,
36
+ "learning_rate": 2.32e-05,
37
+ "loss": 0.9067621231079102,
38
+ "mean_token_accuracy": 0.7337856531143189,
39
+ "num_tokens": 191970.0,
40
+ "step": 30
41
+ },
42
+ {
43
+ "entropy": 0.7395371329039335,
44
+ "epoch": 0.08002000500125031,
45
+ "grad_norm": 1.2754874229431152,
46
+ "learning_rate": 3.12e-05,
47
+ "loss": 0.6462621212005615,
48
+ "mean_token_accuracy": 0.815752174705267,
49
+ "num_tokens": 257775.0,
50
+ "step": 40
51
+ },
52
+ {
53
+ "entropy": 0.5132245156913996,
54
+ "epoch": 0.1000250062515629,
55
+ "grad_norm": 1.6131094694137573,
56
+ "learning_rate": 3.9200000000000004e-05,
57
+ "loss": 0.4761983871459961,
58
+ "mean_token_accuracy": 0.8625759541988373,
59
+ "num_tokens": 319507.0,
60
+ "step": 50
61
+ },
62
+ {
63
+ "entropy": 0.35333055444061756,
64
+ "epoch": 0.12003000750187547,
65
+ "grad_norm": 1.9517418146133423,
66
+ "learning_rate": 4.72e-05,
67
+ "loss": 0.34327900409698486,
68
+ "mean_token_accuracy": 0.9010166764259339,
69
+ "num_tokens": 380952.0,
70
+ "step": 60
71
+ },
72
+ {
73
+ "entropy": 0.2581108913756907,
74
+ "epoch": 0.14003500875218805,
75
+ "grad_norm": 1.8101425170898438,
76
+ "learning_rate": 5.520000000000001e-05,
77
+ "loss": 0.25612337589263917,
78
+ "mean_token_accuracy": 0.9237508125603199,
79
+ "num_tokens": 443849.0,
80
+ "step": 70
81
+ },
82
+ {
83
+ "entropy": 0.23392133321613073,
84
+ "epoch": 0.16004001000250062,
85
+ "grad_norm": 1.4003955125808716,
86
+ "learning_rate": 6.32e-05,
87
+ "loss": 0.20029537677764891,
88
+ "mean_token_accuracy": 0.940458069741726,
89
+ "num_tokens": 505868.0,
90
+ "step": 80
91
+ },
92
+ {
93
+ "entropy": 0.1773814239539206,
94
+ "epoch": 0.1800450112528132,
95
+ "grad_norm": 1.2149077653884888,
96
+ "learning_rate": 7.12e-05,
97
+ "loss": 0.1597532272338867,
98
+ "mean_token_accuracy": 0.9488845877349377,
99
+ "num_tokens": 570792.0,
100
+ "step": 90
101
+ },
102
+ {
103
+ "entropy": 0.16433264631778002,
104
+ "epoch": 0.2000500125031258,
105
+ "grad_norm": 1.4650107622146606,
106
+ "learning_rate": 7.920000000000001e-05,
107
+ "loss": 0.15449292659759523,
108
+ "mean_token_accuracy": 0.9529499627649785,
109
+ "num_tokens": 634476.0,
110
+ "step": 100
111
+ },
112
+ {
113
+ "entropy": 0.1567419718950987,
114
+ "epoch": 0.22005501375343836,
115
+ "grad_norm": 1.2412768602371216,
116
+ "learning_rate": 8.72e-05,
117
+ "loss": 0.14979339838027955,
118
+ "mean_token_accuracy": 0.9547351159155368,
119
+ "num_tokens": 697928.0,
120
+ "step": 110
121
+ },
122
+ {
123
+ "entropy": 0.14790078573860227,
124
+ "epoch": 0.24006001500375093,
125
+ "grad_norm": 1.0941258668899536,
126
+ "learning_rate": 9.52e-05,
127
+ "loss": 0.15537588596343993,
128
+ "mean_token_accuracy": 0.9548502139747143,
129
+ "num_tokens": 761970.0,
130
+ "step": 120
131
+ },
132
+ {
133
+ "entropy": 0.14068418610841035,
134
+ "epoch": 0.26006501625406353,
135
+ "grad_norm": 1.3730136156082153,
136
+ "learning_rate": 0.0001032,
137
+ "loss": 0.12376174926757813,
138
+ "mean_token_accuracy": 0.9643007285892964,
139
+ "num_tokens": 827125.0,
140
+ "step": 130
141
+ },
142
+ {
143
+ "entropy": 0.10366909941658378,
144
+ "epoch": 0.2800700175043761,
145
+ "grad_norm": 0.859259843826294,
146
+ "learning_rate": 0.00011120000000000002,
147
+ "loss": 0.10419689416885376,
148
+ "mean_token_accuracy": 0.9708971530199051,
149
+ "num_tokens": 890387.0,
150
+ "step": 140
151
+ },
152
+ {
153
+ "entropy": 0.10762261427007616,
154
+ "epoch": 0.30007501875468867,
155
+ "grad_norm": 0.9888283610343933,
156
+ "learning_rate": 0.0001192,
157
+ "loss": 0.09227448105812072,
158
+ "mean_token_accuracy": 0.9713370814919472,
159
+ "num_tokens": 956032.0,
160
+ "step": 150
161
+ },
162
+ {
163
+ "entropy": 0.1130900933407247,
164
+ "epoch": 0.32008002000500124,
165
+ "grad_norm": 0.9545535445213318,
166
+ "learning_rate": 0.0001272,
167
+ "loss": 0.1083062767982483,
168
+ "mean_token_accuracy": 0.9707557298243046,
169
+ "num_tokens": 1020072.0,
170
+ "step": 160
171
+ },
172
+ {
173
+ "entropy": 0.0895556318340823,
174
+ "epoch": 0.3400850212553138,
175
+ "grad_norm": 0.9866153597831726,
176
+ "learning_rate": 0.0001352,
177
+ "loss": 0.07606152892112732,
178
+ "mean_token_accuracy": 0.973467419296503,
179
+ "num_tokens": 1083781.0,
180
+ "step": 170
181
+ },
182
+ {
183
+ "entropy": 0.09170240506064146,
184
+ "epoch": 0.3600900225056264,
185
+ "grad_norm": 1.0180600881576538,
186
+ "learning_rate": 0.0001432,
187
+ "loss": 0.10022411346435547,
188
+ "mean_token_accuracy": 0.9705780848860741,
189
+ "num_tokens": 1145466.0,
190
+ "step": 180
191
+ },
192
+ {
193
+ "entropy": 0.11426927694119512,
194
+ "epoch": 0.380095023755939,
195
+ "grad_norm": 1.129612684249878,
196
+ "learning_rate": 0.00015120000000000002,
197
+ "loss": 0.10711104869842529,
198
+ "mean_token_accuracy": 0.9698869682848453,
199
+ "num_tokens": 1208095.0,
200
+ "step": 190
201
+ },
202
+ {
203
+ "entropy": 0.09287996906787158,
204
+ "epoch": 0.4001000250062516,
205
+ "grad_norm": 0.6177782416343689,
206
+ "learning_rate": 0.00015920000000000002,
207
+ "loss": 0.08630093932151794,
208
+ "mean_token_accuracy": 0.977255067974329,
209
+ "num_tokens": 1273403.0,
210
+ "step": 200
211
+ },
212
+ {
213
+ "entropy": 0.0744264661334455,
214
+ "epoch": 0.42010502625656415,
215
+ "grad_norm": 0.766137957572937,
216
+ "learning_rate": 0.0001672,
217
+ "loss": 0.07393635511398315,
218
+ "mean_token_accuracy": 0.9792576834559441,
219
+ "num_tokens": 1334848.0,
220
+ "step": 210
221
+ },
222
+ {
223
+ "entropy": 0.10690853425767273,
224
+ "epoch": 0.4401100275068767,
225
+ "grad_norm": 0.6585826277732849,
226
+ "learning_rate": 0.0001752,
227
+ "loss": 0.09994233846664428,
228
+ "mean_token_accuracy": 0.972873219102621,
229
+ "num_tokens": 1400976.0,
230
+ "step": 220
231
+ },
232
+ {
233
+ "entropy": 0.06925062141381204,
234
+ "epoch": 0.4601150287571893,
235
+ "grad_norm": 0.7318869233131409,
236
+ "learning_rate": 0.0001832,
237
+ "loss": 0.06631548404693603,
238
+ "mean_token_accuracy": 0.9832681395113468,
239
+ "num_tokens": 1462530.0,
240
+ "step": 230
241
+ },
242
+ {
243
+ "entropy": 0.08352572850417346,
244
+ "epoch": 0.48012003000750186,
245
+ "grad_norm": 0.46499672532081604,
246
+ "learning_rate": 0.0001912,
247
+ "loss": 0.09161096215248107,
248
+ "mean_token_accuracy": 0.9757984310388566,
249
+ "num_tokens": 1523663.0,
250
+ "step": 240
251
+ },
252
+ {
253
+ "entropy": 0.06921031260862946,
254
+ "epoch": 0.5001250312578145,
255
+ "grad_norm": 0.6253766417503357,
256
+ "learning_rate": 0.00019920000000000002,
257
+ "loss": 0.05958831310272217,
258
+ "mean_token_accuracy": 0.982395163923502,
259
+ "num_tokens": 1588657.0,
260
+ "step": 250
261
+ },
262
+ {
263
+ "entropy": 0.06543620774755254,
264
+ "epoch": 0.5201300325081271,
265
+ "grad_norm": 0.6349820494651794,
266
+ "learning_rate": 0.00019999822839757118,
267
+ "loss": 0.06900651454925537,
268
+ "mean_token_accuracy": 0.9801992796361446,
269
+ "num_tokens": 1653260.0,
270
+ "step": 260
271
+ },
272
+ {
273
+ "entropy": 0.06953847317490727,
274
+ "epoch": 0.5401350337584396,
275
+ "grad_norm": 0.6046717166900635,
276
+ "learning_rate": 0.00019999210442038162,
277
+ "loss": 0.06709518432617187,
278
+ "mean_token_accuracy": 0.9797540627419948,
279
+ "num_tokens": 1718687.0,
280
+ "step": 270
281
+ },
282
+ {
283
+ "entropy": 0.08993639135733247,
284
+ "epoch": 0.5601400350087522,
285
+ "grad_norm": 0.7929577231407166,
286
+ "learning_rate": 0.00019998160646461522,
287
+ "loss": 0.09603813290596008,
288
+ "mean_token_accuracy": 0.9745012931525707,
289
+ "num_tokens": 1779338.0,
290
+ "step": 280
291
+ },
292
+ {
293
+ "entropy": 0.06307904429268092,
294
+ "epoch": 0.5801450362590648,
295
+ "grad_norm": 0.3916383683681488,
296
+ "learning_rate": 0.00019996673498948658,
297
+ "loss": 0.06412749290466309,
298
+ "mean_token_accuracy": 0.9828441753983498,
299
+ "num_tokens": 1842872.0,
300
+ "step": 290
301
+ },
302
+ {
303
+ "entropy": 0.05729433842934668,
304
+ "epoch": 0.6001500375093773,
305
+ "grad_norm": 0.38883084058761597,
306
+ "learning_rate": 0.00019994749064552214,
307
+ "loss": 0.051730161905288695,
308
+ "mean_token_accuracy": 0.9857619300484657,
309
+ "num_tokens": 1904154.0,
310
+ "step": 300
311
+ },
312
+ {
313
+ "entropy": 0.03964498438872397,
314
+ "epoch": 0.6201550387596899,
315
+ "grad_norm": 0.6382091045379639,
316
+ "learning_rate": 0.0001999238742745319,
317
+ "loss": 0.042625024914741516,
318
+ "mean_token_accuracy": 0.9874393172562123,
319
+ "num_tokens": 1966060.0,
320
+ "step": 310
321
+ },
322
+ {
323
+ "entropy": 0.05733265266753733,
324
+ "epoch": 0.6401600400100025,
325
+ "grad_norm": 0.6905948519706726,
326
+ "learning_rate": 0.00019989588690957249,
327
+ "loss": 0.06732727885246277,
328
+ "mean_token_accuracy": 0.982555440813303,
329
+ "num_tokens": 2031939.0,
330
+ "step": 320
331
+ },
332
+ {
333
+ "entropy": 0.07143733124248683,
334
+ "epoch": 0.660165041260315,
335
+ "grad_norm": 0.4337286949157715,
336
+ "learning_rate": 0.0001998635297749018,
337
+ "loss": 0.058036553859710696,
338
+ "mean_token_accuracy": 0.9844090364873409,
339
+ "num_tokens": 2095271.0,
340
+ "step": 330
341
+ },
342
+ {
343
+ "entropy": 0.04944599290611222,
344
+ "epoch": 0.6801700425106276,
345
+ "grad_norm": 0.3526521921157837,
346
+ "learning_rate": 0.00019982680428592584,
347
+ "loss": 0.053340816497802736,
348
+ "mean_token_accuracy": 0.9839612312614918,
349
+ "num_tokens": 2158321.0,
350
+ "step": 340
351
+ },
352
+ {
353
+ "entropy": 0.06322509178426117,
354
+ "epoch": 0.7001750437609402,
355
+ "grad_norm": 0.5422689318656921,
356
+ "learning_rate": 0.00019978571204913638,
357
+ "loss": 0.051858377456665036,
358
+ "mean_token_accuracy": 0.9867838315665722,
359
+ "num_tokens": 2225062.0,
360
+ "step": 350
361
+ },
362
+ {
363
+ "entropy": 0.046866965352091935,
364
+ "epoch": 0.7201800450112528,
365
+ "grad_norm": 0.7101793885231018,
366
+ "learning_rate": 0.000199740254862041,
367
+ "loss": 0.05608519911766052,
368
+ "mean_token_accuracy": 0.9851541951298713,
369
+ "num_tokens": 2291157.0,
370
+ "step": 360
371
+ },
372
+ {
373
+ "entropy": 0.06618997184559702,
374
+ "epoch": 0.7401850462615653,
375
+ "grad_norm": 0.6186705827713013,
376
+ "learning_rate": 0.00019969043471308436,
377
+ "loss": 0.056741136312484744,
378
+ "mean_token_accuracy": 0.9844918318092823,
379
+ "num_tokens": 2353621.0,
380
+ "step": 370
381
+ },
382
+ {
383
+ "entropy": 0.052203354140510784,
384
+ "epoch": 0.760190047511878,
385
+ "grad_norm": 0.9665183424949646,
386
+ "learning_rate": 0.00019963625378156114,
387
+ "loss": 0.05278732180595398,
388
+ "mean_token_accuracy": 0.9863728702068328,
389
+ "num_tokens": 2415518.0,
390
+ "step": 380
391
+ },
392
+ {
393
+ "entropy": 0.04799637275282294,
394
+ "epoch": 0.7801950487621906,
395
+ "grad_norm": 0.45738333463668823,
396
+ "learning_rate": 0.00019957771443752083,
397
+ "loss": 0.044900187849998476,
398
+ "mean_token_accuracy": 0.9872696734964848,
399
+ "num_tokens": 2477716.0,
400
+ "step": 390
401
+ },
402
+ {
403
+ "entropy": 0.045062902639620005,
404
+ "epoch": 0.8002000500125032,
405
+ "grad_norm": 0.4040285646915436,
406
+ "learning_rate": 0.00019951481924166396,
407
+ "loss": 0.043438228964805606,
408
+ "mean_token_accuracy": 0.9878960207104683,
409
+ "num_tokens": 2540807.0,
410
+ "step": 400
411
+ },
412
+ {
413
+ "entropy": 0.036265855759847906,
414
+ "epoch": 0.8202050512628157,
415
+ "grad_norm": 0.4411618411540985,
416
+ "learning_rate": 0.0001994475709452301,
417
+ "loss": 0.04615793526172638,
418
+ "mean_token_accuracy": 0.9870041370391845,
419
+ "num_tokens": 2603019.0,
420
+ "step": 410
421
+ },
422
+ {
423
+ "entropy": 0.05851077587576583,
424
+ "epoch": 0.8402100525131283,
425
+ "grad_norm": 0.31862467527389526,
426
+ "learning_rate": 0.0001993759724898777,
427
+ "loss": 0.045778676867485046,
428
+ "mean_token_accuracy": 0.9873291261494159,
429
+ "num_tokens": 2667216.0,
430
+ "step": 420
431
+ },
432
+ {
433
+ "entropy": 0.03860169492545538,
434
+ "epoch": 0.8602150537634409,
435
+ "grad_norm": 0.29344919323921204,
436
+ "learning_rate": 0.00019930002700755507,
437
+ "loss": 0.03828286230564117,
438
+ "mean_token_accuracy": 0.988411296159029,
439
+ "num_tokens": 2729805.0,
440
+ "step": 430
441
+ },
442
+ {
443
+ "entropy": 0.03951184251927771,
444
+ "epoch": 0.8802200550137534,
445
+ "grad_norm": 0.4283672571182251,
446
+ "learning_rate": 0.00019921973782036366,
447
+ "loss": 0.038545310497283936,
448
+ "mean_token_accuracy": 0.9878928653895855,
449
+ "num_tokens": 2793292.0,
450
+ "step": 440
451
+ },
452
+ {
453
+ "entropy": 0.028071055363398045,
454
+ "epoch": 0.900225056264066,
455
+ "grad_norm": 0.7173567414283752,
456
+ "learning_rate": 0.0001991351084404126,
457
+ "loss": 0.03233465254306793,
458
+ "mean_token_accuracy": 0.9912850938737392,
459
+ "num_tokens": 2855909.0,
460
+ "step": 450
461
+ },
462
+ {
463
+ "entropy": 0.04781279567687306,
464
+ "epoch": 0.9202300575143786,
465
+ "grad_norm": 0.3380762040615082,
466
+ "learning_rate": 0.00019904614256966512,
467
+ "loss": 0.040875044465065,
468
+ "mean_token_accuracy": 0.9889427930116653,
469
+ "num_tokens": 2916597.0,
470
+ "step": 460
471
+ },
472
+ {
473
+ "entropy": 0.045654052757890896,
474
+ "epoch": 0.9402350587646912,
475
+ "grad_norm": 0.21747978031635284,
476
+ "learning_rate": 0.0001989528440997767,
477
+ "loss": 0.04227911233901978,
478
+ "mean_token_accuracy": 0.987830163538456,
479
+ "num_tokens": 2977012.0,
480
+ "step": 470
481
+ },
482
+ {
483
+ "entropy": 0.039897680119611326,
484
+ "epoch": 0.9602400600150037,
485
+ "grad_norm": 0.3250534236431122,
486
+ "learning_rate": 0.00019885521711192453,
487
+ "loss": 0.03866271674633026,
488
+ "mean_token_accuracy": 0.9888145305216313,
489
+ "num_tokens": 3039866.0,
490
+ "step": 480
491
+ },
492
+ {
493
+ "entropy": 0.0344677664746996,
494
+ "epoch": 0.9802450612653163,
495
+ "grad_norm": 0.5867555141448975,
496
+ "learning_rate": 0.00019875326587662941,
497
+ "loss": 0.030780380964279173,
498
+ "mean_token_accuracy": 0.9910528786480427,
499
+ "num_tokens": 3102450.0,
500
+ "step": 490
501
+ },
502
+ {
503
+ "entropy": 0.03601513006665473,
504
+ "epoch": 1.0,
505
+ "grad_norm": 0.19166764616966248,
506
+ "learning_rate": 0.00019864699485356866,
507
+ "loss": 0.04189955592155457,
508
+ "mean_token_accuracy": 0.989976388744161,
509
+ "num_tokens": 3162838.0,
510
+ "step": 500
511
+ },
512
+ {
513
+ "epoch": 1.0,
514
+ "eval_entropy": 0.051686967685315256,
515
+ "eval_loss": 0.04872545599937439,
516
+ "eval_mean_token_accuracy": 0.9849747346865164,
517
+ "eval_num_tokens": 3162838.0,
518
+ "eval_runtime": 69.4136,
519
+ "eval_samples_per_second": 14.911,
520
+ "eval_steps_per_second": 7.463,
521
+ "step": 500
522
+ },
523
+ {
524
+ "entropy": 0.04407569046597928,
525
+ "epoch": 1.0200050012503126,
526
+ "grad_norm": 0.2645007371902466,
527
+ "learning_rate": 0.00019853640869138103,
528
+ "loss": 0.03468368649482727,
529
+ "mean_token_accuracy": 0.990016209334135,
530
+ "num_tokens": 3228763.0,
531
+ "step": 510
532
+ },
533
+ {
534
+ "entropy": 0.035535094334045426,
535
+ "epoch": 1.0400100025006251,
536
+ "grad_norm": 0.40270644426345825,
537
+ "learning_rate": 0.00019842151222746357,
538
+ "loss": 0.03389493525028229,
539
+ "mean_token_accuracy": 0.9887583516538143,
540
+ "num_tokens": 3293010.0,
541
+ "step": 520
542
+ },
543
+ {
544
+ "entropy": 0.028601143200648948,
545
+ "epoch": 1.0600150037509377,
546
+ "grad_norm": 0.2681713104248047,
547
+ "learning_rate": 0.00019830231048775977,
548
+ "loss": 0.030898410081863403,
549
+ "mean_token_accuracy": 0.9903686709702015,
550
+ "num_tokens": 3354919.0,
551
+ "step": 530
552
+ },
553
+ {
554
+ "entropy": 0.034907517378451304,
555
+ "epoch": 1.0800200050012503,
556
+ "grad_norm": 0.23451267182826996,
557
+ "learning_rate": 0.00019817880868653993,
558
+ "loss": 0.03407395482063293,
559
+ "mean_token_accuracy": 0.9890573389828206,
560
+ "num_tokens": 3415784.0,
561
+ "step": 540
562
+ },
563
+ {
564
+ "entropy": 0.033313815778819846,
565
+ "epoch": 1.1000250062515629,
566
+ "grad_norm": 0.23950988054275513,
567
+ "learning_rate": 0.0001980510122261729,
568
+ "loss": 0.029421544075012206,
569
+ "mean_token_accuracy": 0.9916689246892929,
570
+ "num_tokens": 3480364.0,
571
+ "step": 550
572
+ },
573
+ {
574
+ "entropy": 0.02486751726246439,
575
+ "epoch": 1.1200300075018754,
576
+ "grad_norm": 0.4551165997982025,
577
+ "learning_rate": 0.00019791892669688988,
578
+ "loss": 0.020369285345077516,
579
+ "mean_token_accuracy": 0.9940513521432877,
580
+ "num_tokens": 3542678.0,
581
+ "step": 560
582
+ },
583
+ {
584
+ "entropy": 0.029150180192664264,
585
+ "epoch": 1.140035008752188,
586
+ "grad_norm": 0.20443572103977203,
587
+ "learning_rate": 0.00019778255787653978,
588
+ "loss": 0.02770337164402008,
589
+ "mean_token_accuracy": 0.9897698886692524,
590
+ "num_tokens": 3609086.0,
591
+ "step": 570
592
+ },
593
+ {
594
+ "entropy": 0.031121585314394906,
595
+ "epoch": 1.1600400100025006,
596
+ "grad_norm": 0.3653818368911743,
597
+ "learning_rate": 0.00019764191173033663,
598
+ "loss": 0.025546741485595704,
599
+ "mean_token_accuracy": 0.9925875566899777,
600
+ "num_tokens": 3670310.0,
601
+ "step": 580
602
+ },
603
+ {
604
+ "entropy": 0.02902457951568067,
605
+ "epoch": 1.1800450112528131,
606
+ "grad_norm": 0.4811317026615143,
607
+ "learning_rate": 0.00019749699441059843,
608
+ "loss": 0.027299800515174867,
609
+ "mean_token_accuracy": 0.9906462356448174,
610
+ "num_tokens": 3735307.0,
611
+ "step": 590
612
+ },
613
+ {
614
+ "entropy": 0.030135014103143475,
615
+ "epoch": 1.2000500125031257,
616
+ "grad_norm": 0.21709105372428894,
617
+ "learning_rate": 0.00019734781225647828,
618
+ "loss": 0.02665548324584961,
619
+ "mean_token_accuracy": 0.9932463668286801,
620
+ "num_tokens": 3797213.0,
621
+ "step": 600
622
+ },
623
+ {
624
+ "entropy": 0.03371675145754125,
625
+ "epoch": 1.2200550137534383,
626
+ "grad_norm": 0.631148636341095,
627
+ "learning_rate": 0.00019719437179368688,
628
+ "loss": 0.03535972833633423,
629
+ "mean_token_accuracy": 0.9895499177277088,
630
+ "num_tokens": 3859400.0,
631
+ "step": 610
632
+ },
633
+ {
634
+ "entropy": 0.028748418507166206,
635
+ "epoch": 1.2400600150037508,
636
+ "grad_norm": 0.42123743891716003,
637
+ "learning_rate": 0.00019703667973420706,
638
+ "loss": 0.027658408880233763,
639
+ "mean_token_accuracy": 0.9910015679895878,
640
+ "num_tokens": 3920848.0,
641
+ "step": 620
642
+ },
643
+ {
644
+ "entropy": 0.029886699473718182,
645
+ "epoch": 1.2600650162540634,
646
+ "grad_norm": 0.23473748564720154,
647
+ "learning_rate": 0.00019687474297600045,
648
+ "loss": 0.028230640292167663,
649
+ "mean_token_accuracy": 0.9922301307320595,
650
+ "num_tokens": 3984529.0,
651
+ "step": 630
652
+ },
653
+ {
654
+ "entropy": 0.03151440276997164,
655
+ "epoch": 1.280070017504376,
656
+ "grad_norm": 0.2611916661262512,
657
+ "learning_rate": 0.00019670856860270542,
658
+ "loss": 0.027012214064598083,
659
+ "mean_token_accuracy": 0.9917375601828098,
660
+ "num_tokens": 4048411.0,
661
+ "step": 640
662
+ },
663
+ {
664
+ "entropy": 0.027149295064737088,
665
+ "epoch": 1.3000750187546886,
666
+ "grad_norm": 0.12378375232219696,
667
+ "learning_rate": 0.0001965381638833274,
668
+ "loss": 0.023584455251693726,
669
+ "mean_token_accuracy": 0.9936468034982682,
670
+ "num_tokens": 4112116.0,
671
+ "step": 650
672
+ },
673
+ {
674
+ "entropy": 0.028390987019520253,
675
+ "epoch": 1.3200800200050011,
676
+ "grad_norm": 0.18841037154197693,
677
+ "learning_rate": 0.00019636353627192082,
678
+ "loss": 0.025832393765449525,
679
+ "mean_token_accuracy": 0.9933209784328938,
680
+ "num_tokens": 4175458.0,
681
+ "step": 660
682
+ },
683
+ {
684
+ "entropy": 0.032682666774780954,
685
+ "epoch": 1.3400850212553137,
686
+ "grad_norm": 0.292369544506073,
687
+ "learning_rate": 0.00019618469340726319,
688
+ "loss": 0.032191649079322815,
689
+ "mean_token_accuracy": 0.9895716637372971,
690
+ "num_tokens": 4236466.0,
691
+ "step": 670
692
+ },
693
+ {
694
+ "entropy": 0.032574003856279884,
695
+ "epoch": 1.3600900225056263,
696
+ "grad_norm": 0.2205415517091751,
697
+ "learning_rate": 0.00019600164311252068,
698
+ "loss": 0.02866535782814026,
699
+ "mean_token_accuracy": 0.991417796164751,
700
+ "num_tokens": 4299775.0,
701
+ "step": 680
702
+ },
703
+ {
704
+ "entropy": 0.04169052811048459,
705
+ "epoch": 1.380095023755939,
706
+ "grad_norm": 0.3276292681694031,
707
+ "learning_rate": 0.00019581439339490624,
708
+ "loss": 0.03524776101112366,
709
+ "mean_token_accuracy": 0.9886757604777813,
710
+ "num_tokens": 4365609.0,
711
+ "step": 690
712
+ },
713
+ {
714
+ "entropy": 0.03188371795695275,
715
+ "epoch": 1.4001000250062516,
716
+ "grad_norm": 0.4103780686855316,
717
+ "learning_rate": 0.0001956229524453291,
718
+ "loss": 0.035996857285499576,
719
+ "mean_token_accuracy": 0.9892666183412075,
720
+ "num_tokens": 4432330.0,
721
+ "step": 700
722
+ },
723
+ {
724
+ "entropy": 0.02871296225930564,
725
+ "epoch": 1.4201050262565642,
726
+ "grad_norm": 0.24775762856006622,
727
+ "learning_rate": 0.00019542732863803662,
728
+ "loss": 0.02328706532716751,
729
+ "mean_token_accuracy": 0.9937438026070595,
730
+ "num_tokens": 4496465.0,
731
+ "step": 710
732
+ },
733
+ {
734
+ "entropy": 0.03388670613931026,
735
+ "epoch": 1.4401100275068768,
736
+ "grad_norm": 0.25881657004356384,
737
+ "learning_rate": 0.00019522753053024787,
738
+ "loss": 0.03300818204879761,
739
+ "mean_token_accuracy": 0.9890970505774022,
740
+ "num_tokens": 4560755.0,
741
+ "step": 720
742
+ },
743
+ {
744
+ "entropy": 0.03670836841047276,
745
+ "epoch": 1.4601150287571893,
746
+ "grad_norm": 0.23787066340446472,
747
+ "learning_rate": 0.00019502356686177926,
748
+ "loss": 0.03214167952537537,
749
+ "mean_token_accuracy": 0.9903383336961269,
750
+ "num_tokens": 4625202.0,
751
+ "step": 730
752
+ },
753
+ {
754
+ "entropy": 0.03299383492558263,
755
+ "epoch": 1.480120030007502,
756
+ "grad_norm": 0.31929901242256165,
757
+ "learning_rate": 0.00019481544655466245,
758
+ "loss": 0.031187814474105836,
759
+ "mean_token_accuracy": 0.9905215993523597,
760
+ "num_tokens": 4692174.0,
761
+ "step": 740
762
+ },
763
+ {
764
+ "entropy": 0.023855643330898603,
765
+ "epoch": 1.5001250312578145,
766
+ "grad_norm": 0.24006661772727966,
767
+ "learning_rate": 0.00019460317871275394,
768
+ "loss": 0.025655516982078554,
769
+ "mean_token_accuracy": 0.9913376875221729,
770
+ "num_tokens": 4753192.0,
771
+ "step": 750
772
+ },
773
+ {
774
+ "entropy": 0.033190094074234365,
775
+ "epoch": 1.520130032508127,
776
+ "grad_norm": 0.2834339737892151,
777
+ "learning_rate": 0.00019438677262133668,
778
+ "loss": 0.03126271665096283,
779
+ "mean_token_accuracy": 0.9907064586877823,
780
+ "num_tokens": 4817884.0,
781
+ "step": 760
782
+ },
783
+ {
784
+ "entropy": 0.03176074127841275,
785
+ "epoch": 1.5401350337584396,
786
+ "grad_norm": 0.4165857136249542,
787
+ "learning_rate": 0.00019416623774671425,
788
+ "loss": 0.033395078778266904,
789
+ "mean_token_accuracy": 0.9902952447533607,
790
+ "num_tokens": 4879601.0,
791
+ "step": 770
792
+ },
793
+ {
794
+ "entropy": 0.03441936054150574,
795
+ "epoch": 1.5601400350087522,
796
+ "grad_norm": 0.5297831296920776,
797
+ "learning_rate": 0.00019394158373579645,
798
+ "loss": 0.02917470932006836,
799
+ "mean_token_accuracy": 0.9917018190026283,
800
+ "num_tokens": 4946026.0,
801
+ "step": 780
802
+ },
803
+ {
804
+ "entropy": 0.029212182367336935,
805
+ "epoch": 1.5801450362590648,
806
+ "grad_norm": 0.3761133849620819,
807
+ "learning_rate": 0.00019371282041567752,
808
+ "loss": 0.02672044336795807,
809
+ "mean_token_accuracy": 0.9930847369134426,
810
+ "num_tokens": 5009097.0,
811
+ "step": 790
812
+ },
813
+ {
814
+ "entropy": 0.03556556548574008,
815
+ "epoch": 1.6001500375093773,
816
+ "grad_norm": 0.2545328438282013,
817
+ "learning_rate": 0.0001934799577932062,
818
+ "loss": 0.0338908702135086,
819
+ "mean_token_accuracy": 0.990411739051342,
820
+ "num_tokens": 5076314.0,
821
+ "step": 800
822
+ },
823
+ {
824
+ "entropy": 0.02515874128730502,
825
+ "epoch": 1.62015503875969,
826
+ "grad_norm": 0.3428667187690735,
827
+ "learning_rate": 0.0001932430060545479,
828
+ "loss": 0.022386419773101806,
829
+ "mean_token_accuracy": 0.9932228110730648,
830
+ "num_tokens": 5135522.0,
831
+ "step": 810
832
+ },
833
+ {
834
+ "entropy": 0.021622967024450192,
835
+ "epoch": 1.6401600400100025,
836
+ "grad_norm": 0.19730041921138763,
837
+ "learning_rate": 0.00019300197556473936,
838
+ "loss": 0.02211230844259262,
839
+ "mean_token_accuracy": 0.993052315711975,
840
+ "num_tokens": 5198651.0,
841
+ "step": 820
842
+ },
843
+ {
844
+ "entropy": 0.022755468662944624,
845
+ "epoch": 1.660165041260315,
846
+ "grad_norm": 0.24043497443199158,
847
+ "learning_rate": 0.00019275687686723497,
848
+ "loss": 0.021593029797077178,
849
+ "mean_token_accuracy": 0.9927247293293476,
850
+ "num_tokens": 5258441.0,
851
+ "step": 830
852
+ },
853
+ {
854
+ "entropy": 0.02988760783628095,
855
+ "epoch": 1.6801700425106276,
856
+ "grad_norm": 0.22126108407974243,
857
+ "learning_rate": 0.0001925077206834458,
858
+ "loss": 0.02743455469608307,
859
+ "mean_token_accuracy": 0.9924947433173656,
860
+ "num_tokens": 5323635.0,
861
+ "step": 840
862
+ },
863
+ {
864
+ "entropy": 0.022633779112948105,
865
+ "epoch": 1.7001750437609402,
866
+ "grad_norm": 0.2528051435947418,
867
+ "learning_rate": 0.00019225451791227052,
868
+ "loss": 0.01915370374917984,
869
+ "mean_token_accuracy": 0.9935295671224594,
870
+ "num_tokens": 5388098.0,
871
+ "step": 850
872
+ },
873
+ {
874
+ "entropy": 0.024347139010205864,
875
+ "epoch": 1.7201800450112528,
876
+ "grad_norm": 0.35426992177963257,
877
+ "learning_rate": 0.00019199727962961852,
878
+ "loss": 0.025216898322105406,
879
+ "mean_token_accuracy": 0.9927972495555878,
880
+ "num_tokens": 5450303.0,
881
+ "step": 860
882
+ },
883
+ {
884
+ "entropy": 0.02343553317186888,
885
+ "epoch": 1.7401850462615653,
886
+ "grad_norm": 0.2603664994239807,
887
+ "learning_rate": 0.00019173601708792566,
888
+ "loss": 0.022424712777137756,
889
+ "mean_token_accuracy": 0.9926920354366302,
890
+ "num_tokens": 5513723.0,
891
+ "step": 870
892
+ },
893
+ {
894
+ "entropy": 0.03952418522676453,
895
+ "epoch": 1.7601900475118781,
896
+ "grad_norm": 0.21528302133083344,
897
+ "learning_rate": 0.0001914707417156619,
898
+ "loss": 0.03777352273464203,
899
+ "mean_token_accuracy": 0.9895162962377071,
900
+ "num_tokens": 5575942.0,
901
+ "step": 880
902
+ },
903
+ {
904
+ "entropy": 0.037056630512233825,
905
+ "epoch": 1.7801950487621907,
906
+ "grad_norm": 0.39761167764663696,
907
+ "learning_rate": 0.00019120146511683142,
908
+ "loss": 0.02916957139968872,
909
+ "mean_token_accuracy": 0.9906649015843868,
910
+ "num_tokens": 5640243.0,
911
+ "step": 890
912
+ },
913
+ {
914
+ "entropy": 0.022086267481790857,
915
+ "epoch": 1.8002000500125033,
916
+ "grad_norm": 0.43694376945495605,
917
+ "learning_rate": 0.00019092819907046493,
918
+ "loss": 0.023631574213504793,
919
+ "mean_token_accuracy": 0.9936031945049763,
920
+ "num_tokens": 5702162.0,
921
+ "step": 900
922
+ },
923
+ {
924
+ "entropy": 0.028572715594782493,
925
+ "epoch": 1.8202050512628158,
926
+ "grad_norm": 0.4214474558830261,
927
+ "learning_rate": 0.00019065095553010458,
928
+ "loss": 0.025940075516700745,
929
+ "mean_token_accuracy": 0.9905624501407146,
930
+ "num_tokens": 5764127.0,
931
+ "step": 910
932
+ },
933
+ {
934
+ "entropy": 0.028239472245331854,
935
+ "epoch": 1.8402100525131284,
936
+ "grad_norm": 0.40316465497016907,
937
+ "learning_rate": 0.00019036974662328096,
938
+ "loss": 0.027613845467567445,
939
+ "mean_token_accuracy": 0.991669587045908,
940
+ "num_tokens": 5827235.0,
941
+ "step": 920
942
+ },
943
+ {
944
+ "entropy": 0.028241146955406294,
945
+ "epoch": 1.860215053763441,
946
+ "grad_norm": 0.524972677230835,
947
+ "learning_rate": 0.0001900845846509827,
948
+ "loss": 0.03177001476287842,
949
+ "mean_token_accuracy": 0.9903169378638268,
950
+ "num_tokens": 5890040.0,
951
+ "step": 930
952
+ },
953
+ {
954
+ "entropy": 0.03260187199921347,
955
+ "epoch": 1.8802200550137536,
956
+ "grad_norm": 0.43904051184654236,
957
+ "learning_rate": 0.00018979548208711817,
958
+ "loss": 0.02139996737241745,
959
+ "mean_token_accuracy": 0.9935068063437938,
960
+ "num_tokens": 5953149.0,
961
+ "step": 940
962
+ },
963
+ {
964
+ "entropy": 0.025593279630993494,
965
+ "epoch": 1.9002250562640661,
966
+ "grad_norm": 0.25154390931129456,
967
+ "learning_rate": 0.00018950245157797014,
968
+ "loss": 0.02489333599805832,
969
+ "mean_token_accuracy": 0.9933448024094105,
970
+ "num_tokens": 6014655.0,
971
+ "step": 950
972
+ },
973
+ {
974
+ "entropy": 0.025231685642211232,
975
+ "epoch": 1.9202300575143787,
976
+ "grad_norm": 0.2318635731935501,
977
+ "learning_rate": 0.00018920550594164238,
978
+ "loss": 0.02543329894542694,
979
+ "mean_token_accuracy": 0.9929649449884892,
980
+ "num_tokens": 6078667.0,
981
+ "step": 960
982
+ },
983
+ {
984
+ "entropy": 0.0415392193797743,
985
+ "epoch": 1.9402350587646913,
986
+ "grad_norm": 0.2020518183708191,
987
+ "learning_rate": 0.00018890465816749896,
988
+ "loss": 0.03582499623298645,
989
+ "mean_token_accuracy": 0.9897747471928596,
990
+ "num_tokens": 6139688.0,
991
+ "step": 970
992
+ },
993
+ {
994
+ "entropy": 0.026439224516798275,
995
+ "epoch": 1.9602400600150038,
996
+ "grad_norm": 0.25302180647850037,
997
+ "learning_rate": 0.00018859992141559615,
998
+ "loss": 0.023522551357746124,
999
+ "mean_token_accuracy": 0.9931276544928551,
1000
+ "num_tokens": 6203193.0,
1001
+ "step": 980
1002
+ },
1003
+ {
1004
+ "entropy": 0.02257391346647637,
1005
+ "epoch": 1.9802450612653164,
1006
+ "grad_norm": 0.37129050493240356,
1007
+ "learning_rate": 0.00018829130901610667,
1008
+ "loss": 0.023089873790740966,
1009
+ "mean_token_accuracy": 0.9930807471275329,
1010
+ "num_tokens": 6265353.0,
1011
+ "step": 990
1012
+ },
1013
+ {
1014
+ "entropy": 0.024826381788765894,
1015
+ "epoch": 2.0,
1016
+ "grad_norm": 0.1416839361190796,
1017
+ "learning_rate": 0.00018797883446873662,
1018
+ "loss": 0.020879687368869783,
1019
+ "mean_token_accuracy": 0.9931328839893583,
1020
+ "num_tokens": 6325676.0,
1021
+ "step": 1000
1022
+ },
1023
+ {
1024
+ "epoch": 2.0,
1025
+ "eval_entropy": 0.028477752043098808,
1026
+ "eval_loss": 0.042340315878391266,
1027
+ "eval_mean_token_accuracy": 0.988268693211456,
1028
+ "eval_num_tokens": 6325676.0,
1029
+ "eval_runtime": 69.2562,
1030
+ "eval_samples_per_second": 14.945,
1031
+ "eval_steps_per_second": 7.479,
1032
+ "step": 1000
1033
+ },
1034
+ {
1035
+ "entropy": 0.021088267347658986,
1036
+ "epoch": 2.0200050012503126,
1037
+ "grad_norm": 0.33506715297698975,
1038
+ "learning_rate": 0.00018766251144213504,
1039
+ "loss": 0.0183292493224144,
1040
+ "mean_token_accuracy": 0.9939773567020893,
1041
+ "num_tokens": 6391016.0,
1042
+ "step": 1010
1043
+ },
1044
+ {
1045
+ "entropy": 0.021377279089938382,
1046
+ "epoch": 2.040010002500625,
1047
+ "grad_norm": 0.10510263592004776,
1048
+ "learning_rate": 0.00018734235377329582,
1049
+ "loss": 0.016541089117527007,
1050
+ "mean_token_accuracy": 0.9943198271095752,
1051
+ "num_tokens": 6455040.0,
1052
+ "step": 1020
1053
+ },
1054
+ {
1055
+ "entropy": 0.01835900279111229,
1056
+ "epoch": 2.0600150037509377,
1057
+ "grad_norm": 0.25538530945777893,
1058
+ "learning_rate": 0.0001870183754669526,
1059
+ "loss": 0.017676208913326264,
1060
+ "mean_token_accuracy": 0.9943479098379612,
1061
+ "num_tokens": 6517983.0,
1062
+ "step": 1030
1063
+ },
1064
+ {
1065
+ "entropy": 0.024707998137455434,
1066
+ "epoch": 2.0800200050012503,
1067
+ "grad_norm": 0.1953907310962677,
1068
+ "learning_rate": 0.00018669059069496594,
1069
+ "loss": 0.018231543898582458,
1070
+ "mean_token_accuracy": 0.9940299488604069,
1071
+ "num_tokens": 6582155.0,
1072
+ "step": 1040
1073
+ },
1074
+ {
1075
+ "entropy": 0.016339628079731484,
1076
+ "epoch": 2.100025006251563,
1077
+ "grad_norm": 0.23519866168498993,
1078
+ "learning_rate": 0.00018635901379570377,
1079
+ "loss": 0.015705856680870055,
1080
+ "mean_token_accuracy": 0.9944866336882114,
1081
+ "num_tokens": 6644403.0,
1082
+ "step": 1050
1083
+ },
1084
+ {
1085
+ "entropy": 0.021186151236179285,
1086
+ "epoch": 2.1200300075018754,
1087
+ "grad_norm": 0.7782704830169678,
1088
+ "learning_rate": 0.00018602365927341375,
1089
+ "loss": 0.0239964097738266,
1090
+ "mean_token_accuracy": 0.9942199148237705,
1091
+ "num_tokens": 6709057.0,
1092
+ "step": 1060
1093
+ },
1094
+ {
1095
+ "entropy": 0.02400836138986051,
1096
+ "epoch": 2.140035008752188,
1097
+ "grad_norm": 0.25117242336273193,
1098
+ "learning_rate": 0.0001856845417975891,
1099
+ "loss": 0.0214329332113266,
1100
+ "mean_token_accuracy": 0.99395372569561,
1101
+ "num_tokens": 6772040.0,
1102
+ "step": 1070
1103
+ },
1104
+ {
1105
+ "entropy": 0.01854798578133341,
1106
+ "epoch": 2.1600400100025006,
1107
+ "grad_norm": 0.24927817285060883,
1108
+ "learning_rate": 0.0001853416762023268,
1109
+ "loss": 0.01707075983285904,
1110
+ "mean_token_accuracy": 0.9950113117694854,
1111
+ "num_tokens": 6835866.0,
1112
+ "step": 1080
1113
+ },
1114
+ {
1115
+ "entropy": 0.019260429358109833,
1116
+ "epoch": 2.180045011252813,
1117
+ "grad_norm": 0.24353672564029694,
1118
+ "learning_rate": 0.00018499507748567873,
1119
+ "loss": 0.014785376191139222,
1120
+ "mean_token_accuracy": 0.9959283553063869,
1121
+ "num_tokens": 6899725.0,
1122
+ "step": 1090
1123
+ },
1124
+ {
1125
+ "entropy": 0.02487965851032641,
1126
+ "epoch": 2.2000500125031257,
1127
+ "grad_norm": 0.288215309381485,
1128
+ "learning_rate": 0.00018464476080899559,
1129
+ "loss": 0.019350311160087584,
1130
+ "mean_token_accuracy": 0.9941258184611798,
1131
+ "num_tokens": 6963141.0,
1132
+ "step": 1100
1133
+ },
1134
+ {
1135
+ "entropy": 0.01580278711626306,
1136
+ "epoch": 2.2200550137534383,
1137
+ "grad_norm": 0.0646059513092041,
1138
+ "learning_rate": 0.00018429074149626363,
1139
+ "loss": 0.017427970468997956,
1140
+ "mean_token_accuracy": 0.9952689491212368,
1141
+ "num_tokens": 7028325.0,
1142
+ "step": 1110
1143
+ },
1144
+ {
1145
+ "entropy": 0.020199327028240077,
1146
+ "epoch": 2.240060015003751,
1147
+ "grad_norm": 0.160948246717453,
1148
+ "learning_rate": 0.0001839330350334345,
1149
+ "loss": 0.01726052612066269,
1150
+ "mean_token_accuracy": 0.9933249458670617,
1151
+ "num_tokens": 7092920.0,
1152
+ "step": 1120
1153
+ },
1154
+ {
1155
+ "entropy": 0.021389624777657445,
1156
+ "epoch": 2.2600650162540634,
1157
+ "grad_norm": 0.2359917163848877,
1158
+ "learning_rate": 0.00018357165706774767,
1159
+ "loss": 0.01608244627714157,
1160
+ "mean_token_accuracy": 0.9940837919712067,
1161
+ "num_tokens": 7160997.0,
1162
+ "step": 1130
1163
+ },
1164
+ {
1165
+ "entropy": 0.02022790874907514,
1166
+ "epoch": 2.280070017504376,
1167
+ "grad_norm": 0.07118914276361465,
1168
+ "learning_rate": 0.00018320662340704609,
1169
+ "loss": 0.020376379787921905,
1170
+ "mean_token_accuracy": 0.9942706093192101,
1171
+ "num_tokens": 7226007.0,
1172
+ "step": 1140
1173
+ },
1174
+ {
1175
+ "entropy": 0.01867675751855131,
1176
+ "epoch": 2.3000750187546886,
1177
+ "grad_norm": 0.25310513377189636,
1178
+ "learning_rate": 0.00018283795001908457,
1179
+ "loss": 0.01612715721130371,
1180
+ "mean_token_accuracy": 0.9942055746912957,
1181
+ "num_tokens": 7287986.0,
1182
+ "step": 1150
1183
+ },
1184
+ {
1185
+ "entropy": 0.015577676898101345,
1186
+ "epoch": 2.320080020005001,
1187
+ "grad_norm": 0.2142266184091568,
1188
+ "learning_rate": 0.0001824656530308315,
1189
+ "loss": 0.016605789959430694,
1190
+ "mean_token_accuracy": 0.9948085315525532,
1191
+ "num_tokens": 7349421.0,
1192
+ "step": 1160
1193
+ },
1194
+ {
1195
+ "entropy": 0.020357475349737798,
1196
+ "epoch": 2.3400850212553137,
1197
+ "grad_norm": 0.07947535812854767,
1198
+ "learning_rate": 0.00018208974872776322,
1199
+ "loss": 0.018196111917495726,
1200
+ "mean_token_accuracy": 0.993843074887991,
1201
+ "num_tokens": 7411941.0,
1202
+ "step": 1170
1203
+ },
1204
+ {
1205
+ "entropy": 0.017023067966511006,
1206
+ "epoch": 2.3600900225056263,
1207
+ "grad_norm": 0.19202570617198944,
1208
+ "learning_rate": 0.00018171025355315164,
1209
+ "loss": 0.016091108322143555,
1210
+ "mean_token_accuracy": 0.9953217662870883,
1211
+ "num_tokens": 7474356.0,
1212
+ "step": 1180
1213
+ },
1214
+ {
1215
+ "entropy": 0.01825423450791277,
1216
+ "epoch": 2.380095023755939,
1217
+ "grad_norm": 0.24460658431053162,
1218
+ "learning_rate": 0.00018132718410734515,
1219
+ "loss": 0.015425822138786316,
1220
+ "mean_token_accuracy": 0.9943965286016464,
1221
+ "num_tokens": 7536565.0,
1222
+ "step": 1190
1223
+ },
1224
+ {
1225
+ "entropy": 0.015542891589575447,
1226
+ "epoch": 2.4001000250062514,
1227
+ "grad_norm": 0.24659694731235504,
1228
+ "learning_rate": 0.00018094055714704225,
1229
+ "loss": 0.013514925539493561,
1230
+ "mean_token_accuracy": 0.9943179704248906,
1231
+ "num_tokens": 7599751.0,
1232
+ "step": 1200
1233
+ },
1234
+ {
1235
+ "entropy": 0.015307287167524919,
1236
+ "epoch": 2.420105026256564,
1237
+ "grad_norm": 0.2550601065158844,
1238
+ "learning_rate": 0.0001805503895845587,
1239
+ "loss": 0.012726837396621704,
1240
+ "mean_token_accuracy": 0.9962130591273308,
1241
+ "num_tokens": 7661520.0,
1242
+ "step": 1210
1243
+ },
1244
+ {
1245
+ "entropy": 0.01749844834121177,
1246
+ "epoch": 2.4401100275068766,
1247
+ "grad_norm": 0.15603283047676086,
1248
+ "learning_rate": 0.00018015669848708767,
1249
+ "loss": 0.012257835268974305,
1250
+ "mean_token_accuracy": 0.9954387851059436,
1251
+ "num_tokens": 7726291.0,
1252
+ "step": 1220
1253
+ },
1254
+ {
1255
+ "entropy": 0.019737370508664753,
1256
+ "epoch": 2.460115028757189,
1257
+ "grad_norm": 0.22746174037456512,
1258
+ "learning_rate": 0.0001797595010759531,
1259
+ "loss": 0.018561355769634247,
1260
+ "mean_token_accuracy": 0.9940820440649987,
1261
+ "num_tokens": 7788766.0,
1262
+ "step": 1230
1263
+ },
1264
+ {
1265
+ "entropy": 0.015929855390277227,
1266
+ "epoch": 2.4801200300075017,
1267
+ "grad_norm": 0.3152976334095001,
1268
+ "learning_rate": 0.0001793588147258565,
1269
+ "loss": 0.013849316537380219,
1270
+ "mean_token_accuracy": 0.9952766291797162,
1271
+ "num_tokens": 7851396.0,
1272
+ "step": 1240
1273
+ },
1274
+ {
1275
+ "entropy": 0.02108022033935413,
1276
+ "epoch": 2.5001250312578147,
1277
+ "grad_norm": 0.2762889862060547,
1278
+ "learning_rate": 0.00017895465696411692,
1279
+ "loss": 0.019299986958503722,
1280
+ "mean_token_accuracy": 0.9940553769469261,
1281
+ "num_tokens": 7915421.0,
1282
+ "step": 1250
1283
+ },
1284
+ {
1285
+ "entropy": 0.020866505106096157,
1286
+ "epoch": 2.520130032508127,
1287
+ "grad_norm": 0.24086585640907288,
1288
+ "learning_rate": 0.00017854704546990408,
1289
+ "loss": 0.020875005424022673,
1290
+ "mean_token_accuracy": 0.9940896175801754,
1291
+ "num_tokens": 7977133.0,
1292
+ "step": 1260
1293
+ },
1294
+ {
1295
+ "entropy": 0.020857046739547514,
1296
+ "epoch": 2.54013503375844,
1297
+ "grad_norm": 0.23781460523605347,
1298
+ "learning_rate": 0.0001781359980734653,
1299
+ "loss": 0.018643879890441896,
1300
+ "mean_token_accuracy": 0.9929048053920269,
1301
+ "num_tokens": 8040191.0,
1302
+ "step": 1270
1303
+ },
1304
+ {
1305
+ "entropy": 0.021198420476866885,
1306
+ "epoch": 2.560140035008752,
1307
+ "grad_norm": 0.2054099142551422,
1308
+ "learning_rate": 0.0001777215327553452,
1309
+ "loss": 0.017947974801063537,
1310
+ "mean_token_accuracy": 0.9934561550617218,
1311
+ "num_tokens": 8100152.0,
1312
+ "step": 1280
1313
+ },
1314
+ {
1315
+ "entropy": 0.018285114454920405,
1316
+ "epoch": 2.580145036259065,
1317
+ "grad_norm": 0.469458669424057,
1318
+ "learning_rate": 0.00017730366764559955,
1319
+ "loss": 0.0151192307472229,
1320
+ "mean_token_accuracy": 0.9951836079359054,
1321
+ "num_tokens": 8162591.0,
1322
+ "step": 1290
1323
+ },
1324
+ {
1325
+ "entropy": 0.018374070005665998,
1326
+ "epoch": 2.600150037509377,
1327
+ "grad_norm": 0.2882782518863678,
1328
+ "learning_rate": 0.00017688242102300192,
1329
+ "loss": 0.0184975802898407,
1330
+ "mean_token_accuracy": 0.9948060251772404,
1331
+ "num_tokens": 8225224.0,
1332
+ "step": 1300
1333
+ },
1334
+ {
1335
+ "entropy": 0.022974171601526906,
1336
+ "epoch": 2.62015503875969,
1337
+ "grad_norm": 0.21924524009227753,
1338
+ "learning_rate": 0.00017645781131424423,
1339
+ "loss": 0.023296315968036652,
1340
+ "mean_token_accuracy": 0.9924322210252285,
1341
+ "num_tokens": 8290274.0,
1342
+ "step": 1310
1343
+ },
1344
+ {
1345
+ "entropy": 0.02169415617827326,
1346
+ "epoch": 2.6401600400100023,
1347
+ "grad_norm": 0.354758620262146,
1348
+ "learning_rate": 0.00017602985709313073,
1349
+ "loss": 0.017917373776435853,
1350
+ "mean_token_accuracy": 0.9948078036308289,
1351
+ "num_tokens": 8354370.0,
1352
+ "step": 1320
1353
+ },
1354
+ {
1355
+ "entropy": 0.016268294051405972,
1356
+ "epoch": 2.6601650412603153,
1357
+ "grad_norm": 0.4151551127433777,
1358
+ "learning_rate": 0.00017559857707976536,
1359
+ "loss": 0.012286465615034103,
1360
+ "mean_token_accuracy": 0.9961770802736283,
1361
+ "num_tokens": 8415022.0,
1362
+ "step": 1330
1363
+ },
1364
+ {
1365
+ "entropy": 0.0170176492218161,
1366
+ "epoch": 2.6801700425106274,
1367
+ "grad_norm": 0.4065402150154114,
1368
+ "learning_rate": 0.0001751639901397331,
1369
+ "loss": 0.01499750316143036,
1370
+ "mean_token_accuracy": 0.9952280893921852,
1371
+ "num_tokens": 8478286.0,
1372
+ "step": 1340
1373
+ },
1374
+ {
1375
+ "entropy": 0.018367627350380646,
1376
+ "epoch": 2.7001750437609404,
1377
+ "grad_norm": 0.1335880309343338,
1378
+ "learning_rate": 0.0001747261152832746,
1379
+ "loss": 0.015018537640571594,
1380
+ "mean_token_accuracy": 0.9949840374290944,
1381
+ "num_tokens": 8541154.0,
1382
+ "step": 1350
1383
+ },
1384
+ {
1385
+ "entropy": 0.019403737914399245,
1386
+ "epoch": 2.7201800450112525,
1387
+ "grad_norm": 0.20553363859653473,
1388
+ "learning_rate": 0.00017428497166445452,
1389
+ "loss": 0.019237272441387177,
1390
+ "mean_token_accuracy": 0.9930156201124192,
1391
+ "num_tokens": 8605361.0,
1392
+ "step": 1360
1393
+ },
1394
+ {
1395
+ "entropy": 0.02298831292137038,
1396
+ "epoch": 2.7401850462615656,
1397
+ "grad_norm": 0.31576329469680786,
1398
+ "learning_rate": 0.00017384057858032393,
1399
+ "loss": 0.019147740304470064,
1400
+ "mean_token_accuracy": 0.9930574476718903,
1401
+ "num_tokens": 8669155.0,
1402
+ "step": 1370
1403
+ },
1404
+ {
1405
+ "entropy": 0.01718737747578416,
1406
+ "epoch": 2.760190047511878,
1407
+ "grad_norm": 0.21389739215373993,
1408
+ "learning_rate": 0.00017339295547007594,
1409
+ "loss": 0.017152118682861327,
1410
+ "mean_token_accuracy": 0.995334691554308,
1411
+ "num_tokens": 8735019.0,
1412
+ "step": 1380
1413
+ },
1414
+ {
1415
+ "entropy": 0.019683032816101332,
1416
+ "epoch": 2.7801950487621907,
1417
+ "grad_norm": 0.26539650559425354,
1418
+ "learning_rate": 0.00017294212191419547,
1419
+ "loss": 0.019429606199264527,
1420
+ "mean_token_accuracy": 0.9936951123178005,
1421
+ "num_tokens": 8800972.0,
1422
+ "step": 1390
1423
+ },
1424
+ {
1425
+ "entropy": 0.018636453218641692,
1426
+ "epoch": 2.8002000500125033,
1427
+ "grad_norm": 0.2084941565990448,
1428
+ "learning_rate": 0.00017248809763360277,
1429
+ "loss": 0.01909356415271759,
1430
+ "mean_token_accuracy": 0.9940625011920929,
1431
+ "num_tokens": 8867596.0,
1432
+ "step": 1400
1433
+ },
1434
+ {
1435
+ "entropy": 0.023316194582730532,
1436
+ "epoch": 2.820205051262816,
1437
+ "grad_norm": 0.21403200924396515,
1438
+ "learning_rate": 0.0001720309024887907,
1439
+ "loss": 0.018046848475933075,
1440
+ "mean_token_accuracy": 0.9939217306673527,
1441
+ "num_tokens": 8927516.0,
1442
+ "step": 1410
1443
+ },
1444
+ {
1445
+ "entropy": 0.01655098560877377,
1446
+ "epoch": 2.8402100525131284,
1447
+ "grad_norm": 0.33148443698883057,
1448
+ "learning_rate": 0.000171570556478956,
1449
+ "loss": 0.018755699694156646,
1450
+ "mean_token_accuracy": 0.9942230053246022,
1451
+ "num_tokens": 8989089.0,
1452
+ "step": 1420
1453
+ },
1454
+ {
1455
+ "entropy": 0.02549898542056326,
1456
+ "epoch": 2.860215053763441,
1457
+ "grad_norm": 0.18521511554718018,
1458
+ "learning_rate": 0.00017110707974112447,
1459
+ "loss": 0.024141687154769897,
1460
+ "mean_token_accuracy": 0.9930150359869003,
1461
+ "num_tokens": 9054435.0,
1462
+ "step": 1430
1463
+ },
1464
+ {
1465
+ "entropy": 0.01900827525241766,
1466
+ "epoch": 2.8802200550137536,
1467
+ "grad_norm": 0.23067928850650787,
1468
+ "learning_rate": 0.0001706404925492701,
1469
+ "loss": 0.020157690346240997,
1470
+ "mean_token_accuracy": 0.9943146407604218,
1471
+ "num_tokens": 9114274.0,
1472
+ "step": 1440
1473
+ },
1474
+ {
1475
+ "entropy": 0.01623811650206335,
1476
+ "epoch": 2.900225056264066,
1477
+ "grad_norm": 0.5235961079597473,
1478
+ "learning_rate": 0.00017017081531342813,
1479
+ "loss": 0.014147150516510009,
1480
+ "mean_token_accuracy": 0.9944271765649318,
1481
+ "num_tokens": 9174841.0,
1482
+ "step": 1450
1483
+ },
1484
+ {
1485
+ "entropy": 0.023971330239146483,
1486
+ "epoch": 2.9202300575143787,
1487
+ "grad_norm": 0.11377973109483719,
1488
+ "learning_rate": 0.00016969806857880241,
1489
+ "loss": 0.022856634855270386,
1490
+ "mean_token_accuracy": 0.9932261377573013,
1491
+ "num_tokens": 9239456.0,
1492
+ "step": 1460
1493
+ },
1494
+ {
1495
+ "entropy": 0.02552748184389202,
1496
+ "epoch": 2.9402350587646913,
1497
+ "grad_norm": 0.12835904955863953,
1498
+ "learning_rate": 0.00016922227302486667,
1499
+ "loss": 0.02486345320940018,
1500
+ "mean_token_accuracy": 0.9918816141784191,
1501
+ "num_tokens": 9305144.0,
1502
+ "step": 1470
1503
+ },
1504
+ {
1505
+ "entropy": 0.019867337332107125,
1506
+ "epoch": 2.960240060015004,
1507
+ "grad_norm": 0.18033206462860107,
1508
+ "learning_rate": 0.00016874344946445974,
1509
+ "loss": 0.018220885097980498,
1510
+ "mean_token_accuracy": 0.9945706635713577,
1511
+ "num_tokens": 9365854.0,
1512
+ "step": 1480
1513
+ },
1514
+ {
1515
+ "entropy": 0.01948691344150575,
1516
+ "epoch": 2.9802450612653164,
1517
+ "grad_norm": 0.2495020180940628,
1518
+ "learning_rate": 0.00016826161884287533,
1519
+ "loss": 0.01618766337633133,
1520
+ "mean_token_accuracy": 0.9956672258675099,
1521
+ "num_tokens": 9427287.0,
1522
+ "step": 1490
1523
+ },
1524
+ {
1525
+ "entropy": 0.022937397798228586,
1526
+ "epoch": 3.0,
1527
+ "grad_norm": 0.2727943956851959,
1528
+ "learning_rate": 0.00016777680223694575,
1529
+ "loss": 0.024902991950511932,
1530
+ "mean_token_accuracy": 0.9909723401069641,
1531
+ "num_tokens": 9488514.0,
1532
+ "step": 1500
1533
+ },
1534
+ {
1535
+ "epoch": 3.0,
1536
+ "eval_entropy": 0.02839457441272365,
1537
+ "eval_loss": 0.03645886108279228,
1538
+ "eval_mean_token_accuracy": 0.9902446437986661,
1539
+ "eval_num_tokens": 9488514.0,
1540
+ "eval_runtime": 69.2921,
1541
+ "eval_samples_per_second": 14.937,
1542
+ "eval_steps_per_second": 7.476,
1543
+ "step": 1500
1544
+ }
1545
+ ],
1546
+ "logging_steps": 10,
1547
+ "max_steps": 5000,
1548
+ "num_input_tokens_seen": 0,
1549
+ "num_train_epochs": 10,
1550
+ "save_steps": 500,
1551
+ "stateful_callbacks": {
1552
+ "EarlyStoppingCallback": {
1553
+ "args": {
1554
+ "early_stopping_patience": 3,
1555
+ "early_stopping_threshold": 0.0
1556
+ },
1557
+ "attributes": {
1558
+ "early_stopping_patience_counter": 0
1559
+ }
1560
+ },
1561
+ "TrainerControl": {
1562
+ "args": {
1563
+ "should_epoch_stop": false,
1564
+ "should_evaluate": false,
1565
+ "should_log": false,
1566
+ "should_save": true,
1567
+ "should_training_stop": false
1568
+ },
1569
+ "attributes": {}
1570
+ }
1571
+ },
1572
+ "total_flos": 2.1295589576933376e+16,
1573
+ "train_batch_size": 2,
1574
+ "trial_name": null,
1575
+ "trial_params": null
1576
+ }
checkpoint-1500/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f689d1de06af64c495598bb567c4f842f4f07ab23b514bdea9a9200122abf7d6
3
+ size 5841
run_metadata.json ADDED
@@ -0,0 +1,3119 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "task": "sql2nosql",
3
+ "model_name": "Salesforce/codegen-350M-multi",
4
+ "device": "cuda:0",
5
+ "checkpoint_run": "v4",
6
+ "adapter_path": "/kaggle/working/models/checkpoints/v4/sql2nosql",
7
+ "train_rows": 7998,
8
+ "eval_rows": 1035,
9
+ "train_loss": 0.04043002005169789,
10
+ "eval_loss": 0.03645886108279228,
11
+ "best_eval_loss": 0.03645886108279228,
12
+ "train_runtime_seconds": 10605.5798,
13
+ "mlflow_run_id": null,
14
+ "filter_stats": {
15
+ "input_rows": 8040,
16
+ "kept_rows": 8040,
17
+ "skipped_rows": 42,
18
+ "skip_reasons": {
19
+ "sequence_too_long": 42
20
+ }
21
+ },
22
+ "token_stats": {
23
+ "rows": 7998,
24
+ "avg_prompt_tokens": 319.43,
25
+ "avg_target_tokens": 75.03,
26
+ "avg_total_tokens": 394.45,
27
+ "max_total_tokens": 1328,
28
+ "skipped_too_long_target": 42,
29
+ "skipped_too_long_prompt": 0
30
+ },
31
+ "log_history": [
32
+ {
33
+ "loss": 1.3475940704345704,
34
+ "grad_norm": 1.7302765846252441,
35
+ "learning_rate": 7.2e-06,
36
+ "entropy": 1.1209821447730064,
37
+ "num_tokens": 65756.0,
38
+ "mean_token_accuracy": 0.6775479473173618,
39
+ "epoch": 0.020005001250312578,
40
+ "step": 10
41
+ },
42
+ {
43
+ "loss": 1.1782958984375,
44
+ "grad_norm": 1.598680853843689,
45
+ "learning_rate": 1.52e-05,
46
+ "entropy": 1.0718491852283478,
47
+ "num_tokens": 127350.0,
48
+ "mean_token_accuracy": 0.6978455670177937,
49
+ "epoch": 0.040010002500625155,
50
+ "step": 20
51
+ },
52
+ {
53
+ "loss": 0.9067621231079102,
54
+ "grad_norm": 1.7650020122528076,
55
+ "learning_rate": 2.32e-05,
56
+ "entropy": 0.9551009342074395,
57
+ "num_tokens": 191970.0,
58
+ "mean_token_accuracy": 0.7337856531143189,
59
+ "epoch": 0.06001500375093773,
60
+ "step": 30
61
+ },
62
+ {
63
+ "loss": 0.6462621212005615,
64
+ "grad_norm": 1.2754874229431152,
65
+ "learning_rate": 3.12e-05,
66
+ "entropy": 0.7395371329039335,
67
+ "num_tokens": 257775.0,
68
+ "mean_token_accuracy": 0.815752174705267,
69
+ "epoch": 0.08002000500125031,
70
+ "step": 40
71
+ },
72
+ {
73
+ "loss": 0.4761983871459961,
74
+ "grad_norm": 1.6131094694137573,
75
+ "learning_rate": 3.9200000000000004e-05,
76
+ "entropy": 0.5132245156913996,
77
+ "num_tokens": 319507.0,
78
+ "mean_token_accuracy": 0.8625759541988373,
79
+ "epoch": 0.1000250062515629,
80
+ "step": 50
81
+ },
82
+ {
83
+ "loss": 0.34327900409698486,
84
+ "grad_norm": 1.9517418146133423,
85
+ "learning_rate": 4.72e-05,
86
+ "entropy": 0.35333055444061756,
87
+ "num_tokens": 380952.0,
88
+ "mean_token_accuracy": 0.9010166764259339,
89
+ "epoch": 0.12003000750187547,
90
+ "step": 60
91
+ },
92
+ {
93
+ "loss": 0.25612337589263917,
94
+ "grad_norm": 1.8101425170898438,
95
+ "learning_rate": 5.520000000000001e-05,
96
+ "entropy": 0.2581108913756907,
97
+ "num_tokens": 443849.0,
98
+ "mean_token_accuracy": 0.9237508125603199,
99
+ "epoch": 0.14003500875218805,
100
+ "step": 70
101
+ },
102
+ {
103
+ "loss": 0.20029537677764891,
104
+ "grad_norm": 1.4003955125808716,
105
+ "learning_rate": 6.32e-05,
106
+ "entropy": 0.23392133321613073,
107
+ "num_tokens": 505868.0,
108
+ "mean_token_accuracy": 0.940458069741726,
109
+ "epoch": 0.16004001000250062,
110
+ "step": 80
111
+ },
112
+ {
113
+ "loss": 0.1597532272338867,
114
+ "grad_norm": 1.2149077653884888,
115
+ "learning_rate": 7.12e-05,
116
+ "entropy": 0.1773814239539206,
117
+ "num_tokens": 570792.0,
118
+ "mean_token_accuracy": 0.9488845877349377,
119
+ "epoch": 0.1800450112528132,
120
+ "step": 90
121
+ },
122
+ {
123
+ "loss": 0.15449292659759523,
124
+ "grad_norm": 1.4650107622146606,
125
+ "learning_rate": 7.920000000000001e-05,
126
+ "entropy": 0.16433264631778002,
127
+ "num_tokens": 634476.0,
128
+ "mean_token_accuracy": 0.9529499627649785,
129
+ "epoch": 0.2000500125031258,
130
+ "step": 100
131
+ },
132
+ {
133
+ "loss": 0.14979339838027955,
134
+ "grad_norm": 1.2412768602371216,
135
+ "learning_rate": 8.72e-05,
136
+ "entropy": 0.1567419718950987,
137
+ "num_tokens": 697928.0,
138
+ "mean_token_accuracy": 0.9547351159155368,
139
+ "epoch": 0.22005501375343836,
140
+ "step": 110
141
+ },
142
+ {
143
+ "loss": 0.15537588596343993,
144
+ "grad_norm": 1.0941258668899536,
145
+ "learning_rate": 9.52e-05,
146
+ "entropy": 0.14790078573860227,
147
+ "num_tokens": 761970.0,
148
+ "mean_token_accuracy": 0.9548502139747143,
149
+ "epoch": 0.24006001500375093,
150
+ "step": 120
151
+ },
152
+ {
153
+ "loss": 0.12376174926757813,
154
+ "grad_norm": 1.3730136156082153,
155
+ "learning_rate": 0.0001032,
156
+ "entropy": 0.14068418610841035,
157
+ "num_tokens": 827125.0,
158
+ "mean_token_accuracy": 0.9643007285892964,
159
+ "epoch": 0.26006501625406353,
160
+ "step": 130
161
+ },
162
+ {
163
+ "loss": 0.10419689416885376,
164
+ "grad_norm": 0.859259843826294,
165
+ "learning_rate": 0.00011120000000000002,
166
+ "entropy": 0.10366909941658378,
167
+ "num_tokens": 890387.0,
168
+ "mean_token_accuracy": 0.9708971530199051,
169
+ "epoch": 0.2800700175043761,
170
+ "step": 140
171
+ },
172
+ {
173
+ "loss": 0.09227448105812072,
174
+ "grad_norm": 0.9888283610343933,
175
+ "learning_rate": 0.0001192,
176
+ "entropy": 0.10762261427007616,
177
+ "num_tokens": 956032.0,
178
+ "mean_token_accuracy": 0.9713370814919472,
179
+ "epoch": 0.30007501875468867,
180
+ "step": 150
181
+ },
182
+ {
183
+ "loss": 0.1083062767982483,
184
+ "grad_norm": 0.9545535445213318,
185
+ "learning_rate": 0.0001272,
186
+ "entropy": 0.1130900933407247,
187
+ "num_tokens": 1020072.0,
188
+ "mean_token_accuracy": 0.9707557298243046,
189
+ "epoch": 0.32008002000500124,
190
+ "step": 160
191
+ },
192
+ {
193
+ "loss": 0.07606152892112732,
194
+ "grad_norm": 0.9866153597831726,
195
+ "learning_rate": 0.0001352,
196
+ "entropy": 0.0895556318340823,
197
+ "num_tokens": 1083781.0,
198
+ "mean_token_accuracy": 0.973467419296503,
199
+ "epoch": 0.3400850212553138,
200
+ "step": 170
201
+ },
202
+ {
203
+ "loss": 0.10022411346435547,
204
+ "grad_norm": 1.0180600881576538,
205
+ "learning_rate": 0.0001432,
206
+ "entropy": 0.09170240506064146,
207
+ "num_tokens": 1145466.0,
208
+ "mean_token_accuracy": 0.9705780848860741,
209
+ "epoch": 0.3600900225056264,
210
+ "step": 180
211
+ },
212
+ {
213
+ "loss": 0.10711104869842529,
214
+ "grad_norm": 1.129612684249878,
215
+ "learning_rate": 0.00015120000000000002,
216
+ "entropy": 0.11426927694119512,
217
+ "num_tokens": 1208095.0,
218
+ "mean_token_accuracy": 0.9698869682848453,
219
+ "epoch": 0.380095023755939,
220
+ "step": 190
221
+ },
222
+ {
223
+ "loss": 0.08630093932151794,
224
+ "grad_norm": 0.6177782416343689,
225
+ "learning_rate": 0.00015920000000000002,
226
+ "entropy": 0.09287996906787158,
227
+ "num_tokens": 1273403.0,
228
+ "mean_token_accuracy": 0.977255067974329,
229
+ "epoch": 0.4001000250062516,
230
+ "step": 200
231
+ },
232
+ {
233
+ "loss": 0.07393635511398315,
234
+ "grad_norm": 0.766137957572937,
235
+ "learning_rate": 0.0001672,
236
+ "entropy": 0.0744264661334455,
237
+ "num_tokens": 1334848.0,
238
+ "mean_token_accuracy": 0.9792576834559441,
239
+ "epoch": 0.42010502625656415,
240
+ "step": 210
241
+ },
242
+ {
243
+ "loss": 0.09994233846664428,
244
+ "grad_norm": 0.6585826277732849,
245
+ "learning_rate": 0.0001752,
246
+ "entropy": 0.10690853425767273,
247
+ "num_tokens": 1400976.0,
248
+ "mean_token_accuracy": 0.972873219102621,
249
+ "epoch": 0.4401100275068767,
250
+ "step": 220
251
+ },
252
+ {
253
+ "loss": 0.06631548404693603,
254
+ "grad_norm": 0.7318869233131409,
255
+ "learning_rate": 0.0001832,
256
+ "entropy": 0.06925062141381204,
257
+ "num_tokens": 1462530.0,
258
+ "mean_token_accuracy": 0.9832681395113468,
259
+ "epoch": 0.4601150287571893,
260
+ "step": 230
261
+ },
262
+ {
263
+ "loss": 0.09161096215248107,
264
+ "grad_norm": 0.46499672532081604,
265
+ "learning_rate": 0.0001912,
266
+ "entropy": 0.08352572850417346,
267
+ "num_tokens": 1523663.0,
268
+ "mean_token_accuracy": 0.9757984310388566,
269
+ "epoch": 0.48012003000750186,
270
+ "step": 240
271
+ },
272
+ {
273
+ "loss": 0.05958831310272217,
274
+ "grad_norm": 0.6253766417503357,
275
+ "learning_rate": 0.00019920000000000002,
276
+ "entropy": 0.06921031260862946,
277
+ "num_tokens": 1588657.0,
278
+ "mean_token_accuracy": 0.982395163923502,
279
+ "epoch": 0.5001250312578145,
280
+ "step": 250
281
+ },
282
+ {
283
+ "loss": 0.06900651454925537,
284
+ "grad_norm": 0.6349820494651794,
285
+ "learning_rate": 0.00019999822839757118,
286
+ "entropy": 0.06543620774755254,
287
+ "num_tokens": 1653260.0,
288
+ "mean_token_accuracy": 0.9801992796361446,
289
+ "epoch": 0.5201300325081271,
290
+ "step": 260
291
+ },
292
+ {
293
+ "loss": 0.06709518432617187,
294
+ "grad_norm": 0.6046717166900635,
295
+ "learning_rate": 0.00019999210442038162,
296
+ "entropy": 0.06953847317490727,
297
+ "num_tokens": 1718687.0,
298
+ "mean_token_accuracy": 0.9797540627419948,
299
+ "epoch": 0.5401350337584396,
300
+ "step": 270
301
+ },
302
+ {
303
+ "loss": 0.09603813290596008,
304
+ "grad_norm": 0.7929577231407166,
305
+ "learning_rate": 0.00019998160646461522,
306
+ "entropy": 0.08993639135733247,
307
+ "num_tokens": 1779338.0,
308
+ "mean_token_accuracy": 0.9745012931525707,
309
+ "epoch": 0.5601400350087522,
310
+ "step": 280
311
+ },
312
+ {
313
+ "loss": 0.06412749290466309,
314
+ "grad_norm": 0.3916383683681488,
315
+ "learning_rate": 0.00019996673498948658,
316
+ "entropy": 0.06307904429268092,
317
+ "num_tokens": 1842872.0,
318
+ "mean_token_accuracy": 0.9828441753983498,
319
+ "epoch": 0.5801450362590648,
320
+ "step": 290
321
+ },
322
+ {
323
+ "loss": 0.051730161905288695,
324
+ "grad_norm": 0.38883084058761597,
325
+ "learning_rate": 0.00019994749064552214,
326
+ "entropy": 0.05729433842934668,
327
+ "num_tokens": 1904154.0,
328
+ "mean_token_accuracy": 0.9857619300484657,
329
+ "epoch": 0.6001500375093773,
330
+ "step": 300
331
+ },
332
+ {
333
+ "loss": 0.042625024914741516,
334
+ "grad_norm": 0.6382091045379639,
335
+ "learning_rate": 0.0001999238742745319,
336
+ "entropy": 0.03964498438872397,
337
+ "num_tokens": 1966060.0,
338
+ "mean_token_accuracy": 0.9874393172562123,
339
+ "epoch": 0.6201550387596899,
340
+ "step": 310
341
+ },
342
+ {
343
+ "loss": 0.06732727885246277,
344
+ "grad_norm": 0.6905948519706726,
345
+ "learning_rate": 0.00019989588690957249,
346
+ "entropy": 0.05733265266753733,
347
+ "num_tokens": 2031939.0,
348
+ "mean_token_accuracy": 0.982555440813303,
349
+ "epoch": 0.6401600400100025,
350
+ "step": 320
351
+ },
352
+ {
353
+ "loss": 0.058036553859710696,
354
+ "grad_norm": 0.4337286949157715,
355
+ "learning_rate": 0.0001998635297749018,
356
+ "entropy": 0.07143733124248683,
357
+ "num_tokens": 2095271.0,
358
+ "mean_token_accuracy": 0.9844090364873409,
359
+ "epoch": 0.660165041260315,
360
+ "step": 330
361
+ },
362
+ {
363
+ "loss": 0.053340816497802736,
364
+ "grad_norm": 0.3526521921157837,
365
+ "learning_rate": 0.00019982680428592584,
366
+ "entropy": 0.04944599290611222,
367
+ "num_tokens": 2158321.0,
368
+ "mean_token_accuracy": 0.9839612312614918,
369
+ "epoch": 0.6801700425106276,
370
+ "step": 340
371
+ },
372
+ {
373
+ "loss": 0.051858377456665036,
374
+ "grad_norm": 0.5422689318656921,
375
+ "learning_rate": 0.00019978571204913638,
376
+ "entropy": 0.06322509178426117,
377
+ "num_tokens": 2225062.0,
378
+ "mean_token_accuracy": 0.9867838315665722,
379
+ "epoch": 0.7001750437609402,
380
+ "step": 350
381
+ },
382
+ {
383
+ "loss": 0.05608519911766052,
384
+ "grad_norm": 0.7101793885231018,
385
+ "learning_rate": 0.000199740254862041,
386
+ "entropy": 0.046866965352091935,
387
+ "num_tokens": 2291157.0,
388
+ "mean_token_accuracy": 0.9851541951298713,
389
+ "epoch": 0.7201800450112528,
390
+ "step": 360
391
+ },
392
+ {
393
+ "loss": 0.056741136312484744,
394
+ "grad_norm": 0.6186705827713013,
395
+ "learning_rate": 0.00019969043471308436,
396
+ "entropy": 0.06618997184559702,
397
+ "num_tokens": 2353621.0,
398
+ "mean_token_accuracy": 0.9844918318092823,
399
+ "epoch": 0.7401850462615653,
400
+ "step": 370
401
+ },
402
+ {
403
+ "loss": 0.05278732180595398,
404
+ "grad_norm": 0.9665183424949646,
405
+ "learning_rate": 0.00019963625378156114,
406
+ "entropy": 0.052203354140510784,
407
+ "num_tokens": 2415518.0,
408
+ "mean_token_accuracy": 0.9863728702068328,
409
+ "epoch": 0.760190047511878,
410
+ "step": 380
411
+ },
412
+ {
413
+ "loss": 0.044900187849998476,
414
+ "grad_norm": 0.45738333463668823,
415
+ "learning_rate": 0.00019957771443752083,
416
+ "entropy": 0.04799637275282294,
417
+ "num_tokens": 2477716.0,
418
+ "mean_token_accuracy": 0.9872696734964848,
419
+ "epoch": 0.7801950487621906,
420
+ "step": 390
421
+ },
422
+ {
423
+ "loss": 0.043438228964805606,
424
+ "grad_norm": 0.4040285646915436,
425
+ "learning_rate": 0.00019951481924166396,
426
+ "entropy": 0.045062902639620005,
427
+ "num_tokens": 2540807.0,
428
+ "mean_token_accuracy": 0.9878960207104683,
429
+ "epoch": 0.8002000500125032,
430
+ "step": 400
431
+ },
432
+ {
433
+ "loss": 0.04615793526172638,
434
+ "grad_norm": 0.4411618411540985,
435
+ "learning_rate": 0.0001994475709452301,
436
+ "entropy": 0.036265855759847906,
437
+ "num_tokens": 2603019.0,
438
+ "mean_token_accuracy": 0.9870041370391845,
439
+ "epoch": 0.8202050512628157,
440
+ "step": 410
441
+ },
442
+ {
443
+ "loss": 0.045778676867485046,
444
+ "grad_norm": 0.31862467527389526,
445
+ "learning_rate": 0.0001993759724898777,
446
+ "entropy": 0.05851077587576583,
447
+ "num_tokens": 2667216.0,
448
+ "mean_token_accuracy": 0.9873291261494159,
449
+ "epoch": 0.8402100525131283,
450
+ "step": 420
451
+ },
452
+ {
453
+ "loss": 0.03828286230564117,
454
+ "grad_norm": 0.29344919323921204,
455
+ "learning_rate": 0.00019930002700755507,
456
+ "entropy": 0.03860169492545538,
457
+ "num_tokens": 2729805.0,
458
+ "mean_token_accuracy": 0.988411296159029,
459
+ "epoch": 0.8602150537634409,
460
+ "step": 430
461
+ },
462
+ {
463
+ "loss": 0.038545310497283936,
464
+ "grad_norm": 0.4283672571182251,
465
+ "learning_rate": 0.00019921973782036366,
466
+ "entropy": 0.03951184251927771,
467
+ "num_tokens": 2793292.0,
468
+ "mean_token_accuracy": 0.9878928653895855,
469
+ "epoch": 0.8802200550137534,
470
+ "step": 440
471
+ },
472
+ {
473
+ "loss": 0.03233465254306793,
474
+ "grad_norm": 0.7173567414283752,
475
+ "learning_rate": 0.0001991351084404126,
476
+ "entropy": 0.028071055363398045,
477
+ "num_tokens": 2855909.0,
478
+ "mean_token_accuracy": 0.9912850938737392,
479
+ "epoch": 0.900225056264066,
480
+ "step": 450
481
+ },
482
+ {
483
+ "loss": 0.040875044465065,
484
+ "grad_norm": 0.3380762040615082,
485
+ "learning_rate": 0.00019904614256966512,
486
+ "entropy": 0.04781279567687306,
487
+ "num_tokens": 2916597.0,
488
+ "mean_token_accuracy": 0.9889427930116653,
489
+ "epoch": 0.9202300575143786,
490
+ "step": 460
491
+ },
492
+ {
493
+ "loss": 0.04227911233901978,
494
+ "grad_norm": 0.21747978031635284,
495
+ "learning_rate": 0.0001989528440997767,
496
+ "entropy": 0.045654052757890896,
497
+ "num_tokens": 2977012.0,
498
+ "mean_token_accuracy": 0.987830163538456,
499
+ "epoch": 0.9402350587646912,
500
+ "step": 470
501
+ },
502
+ {
503
+ "loss": 0.03866271674633026,
504
+ "grad_norm": 0.3250534236431122,
505
+ "learning_rate": 0.00019885521711192453,
506
+ "entropy": 0.039897680119611326,
507
+ "num_tokens": 3039866.0,
508
+ "mean_token_accuracy": 0.9888145305216313,
509
+ "epoch": 0.9602400600150037,
510
+ "step": 480
511
+ },
512
+ {
513
+ "loss": 0.030780380964279173,
514
+ "grad_norm": 0.5867555141448975,
515
+ "learning_rate": 0.00019875326587662941,
516
+ "entropy": 0.0344677664746996,
517
+ "num_tokens": 3102450.0,
518
+ "mean_token_accuracy": 0.9910528786480427,
519
+ "epoch": 0.9802450612653163,
520
+ "step": 490
521
+ },
522
+ {
523
+ "loss": 0.04189955592155457,
524
+ "grad_norm": 0.19166764616966248,
525
+ "learning_rate": 0.00019864699485356866,
526
+ "entropy": 0.03601513006665473,
527
+ "num_tokens": 3162838.0,
528
+ "mean_token_accuracy": 0.989976388744161,
529
+ "epoch": 1.0,
530
+ "step": 500
531
+ },
532
+ {
533
+ "eval_loss": 0.04872545599937439,
534
+ "eval_runtime": 69.4136,
535
+ "eval_samples_per_second": 14.911,
536
+ "eval_steps_per_second": 7.463,
537
+ "eval_entropy": 0.051686967685315256,
538
+ "eval_num_tokens": 3162838.0,
539
+ "eval_mean_token_accuracy": 0.9849747346865164,
540
+ "epoch": 1.0,
541
+ "step": 500
542
+ },
543
+ {
544
+ "loss": 0.03468368649482727,
545
+ "grad_norm": 0.2645007371902466,
546
+ "learning_rate": 0.00019853640869138103,
547
+ "entropy": 0.04407569046597928,
548
+ "num_tokens": 3228763.0,
549
+ "mean_token_accuracy": 0.990016209334135,
550
+ "epoch": 1.0200050012503126,
551
+ "step": 510
552
+ },
553
+ {
554
+ "loss": 0.03389493525028229,
555
+ "grad_norm": 0.40270644426345825,
556
+ "learning_rate": 0.00019842151222746357,
557
+ "entropy": 0.035535094334045426,
558
+ "num_tokens": 3293010.0,
559
+ "mean_token_accuracy": 0.9887583516538143,
560
+ "epoch": 1.0400100025006251,
561
+ "step": 520
562
+ },
563
+ {
564
+ "loss": 0.030898410081863403,
565
+ "grad_norm": 0.2681713104248047,
566
+ "learning_rate": 0.00019830231048775977,
567
+ "entropy": 0.028601143200648948,
568
+ "num_tokens": 3354919.0,
569
+ "mean_token_accuracy": 0.9903686709702015,
570
+ "epoch": 1.0600150037509377,
571
+ "step": 530
572
+ },
573
+ {
574
+ "loss": 0.03407395482063293,
575
+ "grad_norm": 0.23451267182826996,
576
+ "learning_rate": 0.00019817880868653993,
577
+ "entropy": 0.034907517378451304,
578
+ "num_tokens": 3415784.0,
579
+ "mean_token_accuracy": 0.9890573389828206,
580
+ "epoch": 1.0800200050012503,
581
+ "step": 540
582
+ },
583
+ {
584
+ "loss": 0.029421544075012206,
585
+ "grad_norm": 0.23950988054275513,
586
+ "learning_rate": 0.0001980510122261729,
587
+ "entropy": 0.033313815778819846,
588
+ "num_tokens": 3480364.0,
589
+ "mean_token_accuracy": 0.9916689246892929,
590
+ "epoch": 1.1000250062515629,
591
+ "step": 550
592
+ },
593
+ {
594
+ "loss": 0.020369285345077516,
595
+ "grad_norm": 0.4551165997982025,
596
+ "learning_rate": 0.00019791892669688988,
597
+ "entropy": 0.02486751726246439,
598
+ "num_tokens": 3542678.0,
599
+ "mean_token_accuracy": 0.9940513521432877,
600
+ "epoch": 1.1200300075018754,
601
+ "step": 560
602
+ },
603
+ {
604
+ "loss": 0.02770337164402008,
605
+ "grad_norm": 0.20443572103977203,
606
+ "learning_rate": 0.00019778255787653978,
607
+ "entropy": 0.029150180192664264,
608
+ "num_tokens": 3609086.0,
609
+ "mean_token_accuracy": 0.9897698886692524,
610
+ "epoch": 1.140035008752188,
611
+ "step": 570
612
+ },
613
+ {
614
+ "loss": 0.025546741485595704,
615
+ "grad_norm": 0.3653818368911743,
616
+ "learning_rate": 0.00019764191173033663,
617
+ "entropy": 0.031121585314394906,
618
+ "num_tokens": 3670310.0,
619
+ "mean_token_accuracy": 0.9925875566899777,
620
+ "epoch": 1.1600400100025006,
621
+ "step": 580
622
+ },
623
+ {
624
+ "loss": 0.027299800515174867,
625
+ "grad_norm": 0.4811317026615143,
626
+ "learning_rate": 0.00019749699441059843,
627
+ "entropy": 0.02902457951568067,
628
+ "num_tokens": 3735307.0,
629
+ "mean_token_accuracy": 0.9906462356448174,
630
+ "epoch": 1.1800450112528131,
631
+ "step": 590
632
+ },
633
+ {
634
+ "loss": 0.02665548324584961,
635
+ "grad_norm": 0.21709105372428894,
636
+ "learning_rate": 0.00019734781225647828,
637
+ "entropy": 0.030135014103143475,
638
+ "num_tokens": 3797213.0,
639
+ "mean_token_accuracy": 0.9932463668286801,
640
+ "epoch": 1.2000500125031257,
641
+ "step": 600
642
+ },
643
+ {
644
+ "loss": 0.03535972833633423,
645
+ "grad_norm": 0.631148636341095,
646
+ "learning_rate": 0.00019719437179368688,
647
+ "entropy": 0.03371675145754125,
648
+ "num_tokens": 3859400.0,
649
+ "mean_token_accuracy": 0.9895499177277088,
650
+ "epoch": 1.2200550137534383,
651
+ "step": 610
652
+ },
653
+ {
654
+ "loss": 0.027658408880233763,
655
+ "grad_norm": 0.42123743891716003,
656
+ "learning_rate": 0.00019703667973420706,
657
+ "entropy": 0.028748418507166206,
658
+ "num_tokens": 3920848.0,
659
+ "mean_token_accuracy": 0.9910015679895878,
660
+ "epoch": 1.2400600150037508,
661
+ "step": 620
662
+ },
663
+ {
664
+ "loss": 0.028230640292167663,
665
+ "grad_norm": 0.23473748564720154,
666
+ "learning_rate": 0.00019687474297600045,
667
+ "entropy": 0.029886699473718182,
668
+ "num_tokens": 3984529.0,
669
+ "mean_token_accuracy": 0.9922301307320595,
670
+ "epoch": 1.2600650162540634,
671
+ "step": 630
672
+ },
673
+ {
674
+ "loss": 0.027012214064598083,
675
+ "grad_norm": 0.2611916661262512,
676
+ "learning_rate": 0.00019670856860270542,
677
+ "entropy": 0.03151440276997164,
678
+ "num_tokens": 4048411.0,
679
+ "mean_token_accuracy": 0.9917375601828098,
680
+ "epoch": 1.280070017504376,
681
+ "step": 640
682
+ },
683
+ {
684
+ "loss": 0.023584455251693726,
685
+ "grad_norm": 0.12378375232219696,
686
+ "learning_rate": 0.0001965381638833274,
687
+ "entropy": 0.027149295064737088,
688
+ "num_tokens": 4112116.0,
689
+ "mean_token_accuracy": 0.9936468034982682,
690
+ "epoch": 1.3000750187546886,
691
+ "step": 650
692
+ },
693
+ {
694
+ "loss": 0.025832393765449525,
695
+ "grad_norm": 0.18841037154197693,
696
+ "learning_rate": 0.00019636353627192082,
697
+ "entropy": 0.028390987019520253,
698
+ "num_tokens": 4175458.0,
699
+ "mean_token_accuracy": 0.9933209784328938,
700
+ "epoch": 1.3200800200050011,
701
+ "step": 660
702
+ },
703
+ {
704
+ "loss": 0.032191649079322815,
705
+ "grad_norm": 0.292369544506073,
706
+ "learning_rate": 0.00019618469340726319,
707
+ "entropy": 0.032682666774780954,
708
+ "num_tokens": 4236466.0,
709
+ "mean_token_accuracy": 0.9895716637372971,
710
+ "epoch": 1.3400850212553137,
711
+ "step": 670
712
+ },
713
+ {
714
+ "loss": 0.02866535782814026,
715
+ "grad_norm": 0.2205415517091751,
716
+ "learning_rate": 0.00019600164311252068,
717
+ "entropy": 0.032574003856279884,
718
+ "num_tokens": 4299775.0,
719
+ "mean_token_accuracy": 0.991417796164751,
720
+ "epoch": 1.3600900225056263,
721
+ "step": 680
722
+ },
723
+ {
724
+ "loss": 0.03524776101112366,
725
+ "grad_norm": 0.3276292681694031,
726
+ "learning_rate": 0.00019581439339490624,
727
+ "entropy": 0.04169052811048459,
728
+ "num_tokens": 4365609.0,
729
+ "mean_token_accuracy": 0.9886757604777813,
730
+ "epoch": 1.380095023755939,
731
+ "step": 690
732
+ },
733
+ {
734
+ "loss": 0.035996857285499576,
735
+ "grad_norm": 0.4103780686855316,
736
+ "learning_rate": 0.0001956229524453291,
737
+ "entropy": 0.03188371795695275,
738
+ "num_tokens": 4432330.0,
739
+ "mean_token_accuracy": 0.9892666183412075,
740
+ "epoch": 1.4001000250062516,
741
+ "step": 700
742
+ },
743
+ {
744
+ "loss": 0.02328706532716751,
745
+ "grad_norm": 0.24775762856006622,
746
+ "learning_rate": 0.00019542732863803662,
747
+ "entropy": 0.02871296225930564,
748
+ "num_tokens": 4496465.0,
749
+ "mean_token_accuracy": 0.9937438026070595,
750
+ "epoch": 1.4201050262565642,
751
+ "step": 710
752
+ },
753
+ {
754
+ "loss": 0.03300818204879761,
755
+ "grad_norm": 0.25881657004356384,
756
+ "learning_rate": 0.00019522753053024787,
757
+ "entropy": 0.03388670613931026,
758
+ "num_tokens": 4560755.0,
759
+ "mean_token_accuracy": 0.9890970505774022,
760
+ "epoch": 1.4401100275068768,
761
+ "step": 720
762
+ },
763
+ {
764
+ "loss": 0.03214167952537537,
765
+ "grad_norm": 0.23787066340446472,
766
+ "learning_rate": 0.00019502356686177926,
767
+ "entropy": 0.03670836841047276,
768
+ "num_tokens": 4625202.0,
769
+ "mean_token_accuracy": 0.9903383336961269,
770
+ "epoch": 1.4601150287571893,
771
+ "step": 730
772
+ },
773
+ {
774
+ "loss": 0.031187814474105836,
775
+ "grad_norm": 0.31929901242256165,
776
+ "learning_rate": 0.00019481544655466245,
777
+ "entropy": 0.03299383492558263,
778
+ "num_tokens": 4692174.0,
779
+ "mean_token_accuracy": 0.9905215993523597,
780
+ "epoch": 1.480120030007502,
781
+ "step": 740
782
+ },
783
+ {
784
+ "loss": 0.025655516982078554,
785
+ "grad_norm": 0.24006661772727966,
786
+ "learning_rate": 0.00019460317871275394,
787
+ "entropy": 0.023855643330898603,
788
+ "num_tokens": 4753192.0,
789
+ "mean_token_accuracy": 0.9913376875221729,
790
+ "epoch": 1.5001250312578145,
791
+ "step": 750
792
+ },
793
+ {
794
+ "loss": 0.03126271665096283,
795
+ "grad_norm": 0.2834339737892151,
796
+ "learning_rate": 0.00019438677262133668,
797
+ "entropy": 0.033190094074234365,
798
+ "num_tokens": 4817884.0,
799
+ "mean_token_accuracy": 0.9907064586877823,
800
+ "epoch": 1.520130032508127,
801
+ "step": 760
802
+ },
803
+ {
804
+ "loss": 0.033395078778266904,
805
+ "grad_norm": 0.4165857136249542,
806
+ "learning_rate": 0.00019416623774671425,
807
+ "entropy": 0.03176074127841275,
808
+ "num_tokens": 4879601.0,
809
+ "mean_token_accuracy": 0.9902952447533607,
810
+ "epoch": 1.5401350337584396,
811
+ "step": 770
812
+ },
813
+ {
814
+ "loss": 0.02917470932006836,
815
+ "grad_norm": 0.5297831296920776,
816
+ "learning_rate": 0.00019394158373579645,
817
+ "entropy": 0.03441936054150574,
818
+ "num_tokens": 4946026.0,
819
+ "mean_token_accuracy": 0.9917018190026283,
820
+ "epoch": 1.5601400350087522,
821
+ "step": 780
822
+ },
823
+ {
824
+ "loss": 0.02672044336795807,
825
+ "grad_norm": 0.3761133849620819,
826
+ "learning_rate": 0.00019371282041567752,
827
+ "entropy": 0.029212182367336935,
828
+ "num_tokens": 5009097.0,
829
+ "mean_token_accuracy": 0.9930847369134426,
830
+ "epoch": 1.5801450362590648,
831
+ "step": 790
832
+ },
833
+ {
834
+ "loss": 0.0338908702135086,
835
+ "grad_norm": 0.2545328438282013,
836
+ "learning_rate": 0.0001934799577932062,
837
+ "entropy": 0.03556556548574008,
838
+ "num_tokens": 5076314.0,
839
+ "mean_token_accuracy": 0.990411739051342,
840
+ "epoch": 1.6001500375093773,
841
+ "step": 800
842
+ },
843
+ {
844
+ "loss": 0.022386419773101806,
845
+ "grad_norm": 0.3428667187690735,
846
+ "learning_rate": 0.0001932430060545479,
847
+ "entropy": 0.02515874128730502,
848
+ "num_tokens": 5135522.0,
849
+ "mean_token_accuracy": 0.9932228110730648,
850
+ "epoch": 1.62015503875969,
851
+ "step": 810
852
+ },
853
+ {
854
+ "loss": 0.02211230844259262,
855
+ "grad_norm": 0.19730041921138763,
856
+ "learning_rate": 0.00019300197556473936,
857
+ "entropy": 0.021622967024450192,
858
+ "num_tokens": 5198651.0,
859
+ "mean_token_accuracy": 0.993052315711975,
860
+ "epoch": 1.6401600400100025,
861
+ "step": 820
862
+ },
863
+ {
864
+ "loss": 0.021593029797077178,
865
+ "grad_norm": 0.24043497443199158,
866
+ "learning_rate": 0.00019275687686723497,
867
+ "entropy": 0.022755468662944624,
868
+ "num_tokens": 5258441.0,
869
+ "mean_token_accuracy": 0.9927247293293476,
870
+ "epoch": 1.660165041260315,
871
+ "step": 830
872
+ },
873
+ {
874
+ "loss": 0.02743455469608307,
875
+ "grad_norm": 0.22126108407974243,
876
+ "learning_rate": 0.0001925077206834458,
877
+ "entropy": 0.02988760783628095,
878
+ "num_tokens": 5323635.0,
879
+ "mean_token_accuracy": 0.9924947433173656,
880
+ "epoch": 1.6801700425106276,
881
+ "step": 840
882
+ },
883
+ {
884
+ "loss": 0.01915370374917984,
885
+ "grad_norm": 0.2528051435947418,
886
+ "learning_rate": 0.00019225451791227052,
887
+ "entropy": 0.022633779112948105,
888
+ "num_tokens": 5388098.0,
889
+ "mean_token_accuracy": 0.9935295671224594,
890
+ "epoch": 1.7001750437609402,
891
+ "step": 850
892
+ },
893
+ {
894
+ "loss": 0.025216898322105406,
895
+ "grad_norm": 0.35426992177963257,
896
+ "learning_rate": 0.00019199727962961852,
897
+ "entropy": 0.024347139010205864,
898
+ "num_tokens": 5450303.0,
899
+ "mean_token_accuracy": 0.9927972495555878,
900
+ "epoch": 1.7201800450112528,
901
+ "step": 860
902
+ },
903
+ {
904
+ "loss": 0.022424712777137756,
905
+ "grad_norm": 0.2603664994239807,
906
+ "learning_rate": 0.00019173601708792566,
907
+ "entropy": 0.02343553317186888,
908
+ "num_tokens": 5513723.0,
909
+ "mean_token_accuracy": 0.9926920354366302,
910
+ "epoch": 1.7401850462615653,
911
+ "step": 870
912
+ },
913
+ {
914
+ "loss": 0.03777352273464203,
915
+ "grad_norm": 0.21528302133083344,
916
+ "learning_rate": 0.0001914707417156619,
917
+ "entropy": 0.03952418522676453,
918
+ "num_tokens": 5575942.0,
919
+ "mean_token_accuracy": 0.9895162962377071,
920
+ "epoch": 1.7601900475118781,
921
+ "step": 880
922
+ },
923
+ {
924
+ "loss": 0.02916957139968872,
925
+ "grad_norm": 0.39761167764663696,
926
+ "learning_rate": 0.00019120146511683142,
927
+ "entropy": 0.037056630512233825,
928
+ "num_tokens": 5640243.0,
929
+ "mean_token_accuracy": 0.9906649015843868,
930
+ "epoch": 1.7801950487621907,
931
+ "step": 890
932
+ },
933
+ {
934
+ "loss": 0.023631574213504793,
935
+ "grad_norm": 0.43694376945495605,
936
+ "learning_rate": 0.00019092819907046493,
937
+ "entropy": 0.022086267481790857,
938
+ "num_tokens": 5702162.0,
939
+ "mean_token_accuracy": 0.9936031945049763,
940
+ "epoch": 1.8002000500125033,
941
+ "step": 900
942
+ },
943
+ {
944
+ "loss": 0.025940075516700745,
945
+ "grad_norm": 0.4214474558830261,
946
+ "learning_rate": 0.00019065095553010458,
947
+ "entropy": 0.028572715594782493,
948
+ "num_tokens": 5764127.0,
949
+ "mean_token_accuracy": 0.9905624501407146,
950
+ "epoch": 1.8202050512628158,
951
+ "step": 910
952
+ },
953
+ {
954
+ "loss": 0.027613845467567445,
955
+ "grad_norm": 0.40316465497016907,
956
+ "learning_rate": 0.00019036974662328096,
957
+ "entropy": 0.028239472245331854,
958
+ "num_tokens": 5827235.0,
959
+ "mean_token_accuracy": 0.991669587045908,
960
+ "epoch": 1.8402100525131284,
961
+ "step": 920
962
+ },
963
+ {
964
+ "loss": 0.03177001476287842,
965
+ "grad_norm": 0.524972677230835,
966
+ "learning_rate": 0.0001900845846509827,
967
+ "entropy": 0.028241146955406294,
968
+ "num_tokens": 5890040.0,
969
+ "mean_token_accuracy": 0.9903169378638268,
970
+ "epoch": 1.860215053763441,
971
+ "step": 930
972
+ },
973
+ {
974
+ "loss": 0.02139996737241745,
975
+ "grad_norm": 0.43904051184654236,
976
+ "learning_rate": 0.00018979548208711817,
977
+ "entropy": 0.03260187199921347,
978
+ "num_tokens": 5953149.0,
979
+ "mean_token_accuracy": 0.9935068063437938,
980
+ "epoch": 1.8802200550137536,
981
+ "step": 940
982
+ },
983
+ {
984
+ "loss": 0.02489333599805832,
985
+ "grad_norm": 0.25154390931129456,
986
+ "learning_rate": 0.00018950245157797014,
987
+ "entropy": 0.025593279630993494,
988
+ "num_tokens": 6014655.0,
989
+ "mean_token_accuracy": 0.9933448024094105,
990
+ "epoch": 1.9002250562640661,
991
+ "step": 950
992
+ },
993
+ {
994
+ "loss": 0.02543329894542694,
995
+ "grad_norm": 0.2318635731935501,
996
+ "learning_rate": 0.00018920550594164238,
997
+ "entropy": 0.025231685642211232,
998
+ "num_tokens": 6078667.0,
999
+ "mean_token_accuracy": 0.9929649449884892,
1000
+ "epoch": 1.9202300575143787,
1001
+ "step": 960
1002
+ },
1003
+ {
1004
+ "loss": 0.03582499623298645,
1005
+ "grad_norm": 0.2020518183708191,
1006
+ "learning_rate": 0.00018890465816749896,
1007
+ "entropy": 0.0415392193797743,
1008
+ "num_tokens": 6139688.0,
1009
+ "mean_token_accuracy": 0.9897747471928596,
1010
+ "epoch": 1.9402350587646913,
1011
+ "step": 970
1012
+ },
1013
+ {
1014
+ "loss": 0.023522551357746124,
1015
+ "grad_norm": 0.25302180647850037,
1016
+ "learning_rate": 0.00018859992141559615,
1017
+ "entropy": 0.026439224516798275,
1018
+ "num_tokens": 6203193.0,
1019
+ "mean_token_accuracy": 0.9931276544928551,
1020
+ "epoch": 1.9602400600150038,
1021
+ "step": 980
1022
+ },
1023
+ {
1024
+ "loss": 0.023089873790740966,
1025
+ "grad_norm": 0.37129050493240356,
1026
+ "learning_rate": 0.00018829130901610667,
1027
+ "entropy": 0.02257391346647637,
1028
+ "num_tokens": 6265353.0,
1029
+ "mean_token_accuracy": 0.9930807471275329,
1030
+ "epoch": 1.9802450612653164,
1031
+ "step": 990
1032
+ },
1033
+ {
1034
+ "loss": 0.020879687368869783,
1035
+ "grad_norm": 0.1416839361190796,
1036
+ "learning_rate": 0.00018797883446873662,
1037
+ "entropy": 0.024826381788765894,
1038
+ "num_tokens": 6325676.0,
1039
+ "mean_token_accuracy": 0.9931328839893583,
1040
+ "epoch": 2.0,
1041
+ "step": 1000
1042
+ },
1043
+ {
1044
+ "eval_loss": 0.042340315878391266,
1045
+ "eval_runtime": 69.2562,
1046
+ "eval_samples_per_second": 14.945,
1047
+ "eval_steps_per_second": 7.479,
1048
+ "eval_entropy": 0.028477752043098808,
1049
+ "eval_num_tokens": 6325676.0,
1050
+ "eval_mean_token_accuracy": 0.988268693211456,
1051
+ "epoch": 2.0,
1052
+ "step": 1000
1053
+ },
1054
+ {
1055
+ "loss": 0.0183292493224144,
1056
+ "grad_norm": 0.33506715297698975,
1057
+ "learning_rate": 0.00018766251144213504,
1058
+ "entropy": 0.021088267347658986,
1059
+ "num_tokens": 6391016.0,
1060
+ "mean_token_accuracy": 0.9939773567020893,
1061
+ "epoch": 2.0200050012503126,
1062
+ "step": 1010
1063
+ },
1064
+ {
1065
+ "loss": 0.016541089117527007,
1066
+ "grad_norm": 0.10510263592004776,
1067
+ "learning_rate": 0.00018734235377329582,
1068
+ "entropy": 0.021377279089938382,
1069
+ "num_tokens": 6455040.0,
1070
+ "mean_token_accuracy": 0.9943198271095752,
1071
+ "epoch": 2.040010002500625,
1072
+ "step": 1020
1073
+ },
1074
+ {
1075
+ "loss": 0.017676208913326264,
1076
+ "grad_norm": 0.25538530945777893,
1077
+ "learning_rate": 0.0001870183754669526,
1078
+ "entropy": 0.01835900279111229,
1079
+ "num_tokens": 6517983.0,
1080
+ "mean_token_accuracy": 0.9943479098379612,
1081
+ "epoch": 2.0600150037509377,
1082
+ "step": 1030
1083
+ },
1084
+ {
1085
+ "loss": 0.018231543898582458,
1086
+ "grad_norm": 0.1953907310962677,
1087
+ "learning_rate": 0.00018669059069496594,
1088
+ "entropy": 0.024707998137455434,
1089
+ "num_tokens": 6582155.0,
1090
+ "mean_token_accuracy": 0.9940299488604069,
1091
+ "epoch": 2.0800200050012503,
1092
+ "step": 1040
1093
+ },
1094
+ {
1095
+ "loss": 0.015705856680870055,
1096
+ "grad_norm": 0.23519866168498993,
1097
+ "learning_rate": 0.00018635901379570377,
1098
+ "entropy": 0.016339628079731484,
1099
+ "num_tokens": 6644403.0,
1100
+ "mean_token_accuracy": 0.9944866336882114,
1101
+ "epoch": 2.100025006251563,
1102
+ "step": 1050
1103
+ },
1104
+ {
1105
+ "loss": 0.0239964097738266,
1106
+ "grad_norm": 0.7782704830169678,
1107
+ "learning_rate": 0.00018602365927341375,
1108
+ "entropy": 0.021186151236179285,
1109
+ "num_tokens": 6709057.0,
1110
+ "mean_token_accuracy": 0.9942199148237705,
1111
+ "epoch": 2.1200300075018754,
1112
+ "step": 1060
1113
+ },
1114
+ {
1115
+ "loss": 0.0214329332113266,
1116
+ "grad_norm": 0.25117242336273193,
1117
+ "learning_rate": 0.0001856845417975891,
1118
+ "entropy": 0.02400836138986051,
1119
+ "num_tokens": 6772040.0,
1120
+ "mean_token_accuracy": 0.99395372569561,
1121
+ "epoch": 2.140035008752188,
1122
+ "step": 1070
1123
+ },
1124
+ {
1125
+ "loss": 0.01707075983285904,
1126
+ "grad_norm": 0.24927817285060883,
1127
+ "learning_rate": 0.0001853416762023268,
1128
+ "entropy": 0.01854798578133341,
1129
+ "num_tokens": 6835866.0,
1130
+ "mean_token_accuracy": 0.9950113117694854,
1131
+ "epoch": 2.1600400100025006,
1132
+ "step": 1080
1133
+ },
1134
+ {
1135
+ "loss": 0.014785376191139222,
1136
+ "grad_norm": 0.24353672564029694,
1137
+ "learning_rate": 0.00018499507748567873,
1138
+ "entropy": 0.019260429358109833,
1139
+ "num_tokens": 6899725.0,
1140
+ "mean_token_accuracy": 0.9959283553063869,
1141
+ "epoch": 2.180045011252813,
1142
+ "step": 1090
1143
+ },
1144
+ {
1145
+ "loss": 0.019350311160087584,
1146
+ "grad_norm": 0.288215309381485,
1147
+ "learning_rate": 0.00018464476080899559,
1148
+ "entropy": 0.02487965851032641,
1149
+ "num_tokens": 6963141.0,
1150
+ "mean_token_accuracy": 0.9941258184611798,
1151
+ "epoch": 2.2000500125031257,
1152
+ "step": 1100
1153
+ },
1154
+ {
1155
+ "loss": 0.017427970468997956,
1156
+ "grad_norm": 0.0646059513092041,
1157
+ "learning_rate": 0.00018429074149626363,
1158
+ "entropy": 0.01580278711626306,
1159
+ "num_tokens": 7028325.0,
1160
+ "mean_token_accuracy": 0.9952689491212368,
1161
+ "epoch": 2.2200550137534383,
1162
+ "step": 1110
1163
+ },
1164
+ {
1165
+ "loss": 0.01726052612066269,
1166
+ "grad_norm": 0.160948246717453,
1167
+ "learning_rate": 0.0001839330350334345,
1168
+ "entropy": 0.020199327028240077,
1169
+ "num_tokens": 7092920.0,
1170
+ "mean_token_accuracy": 0.9933249458670617,
1171
+ "epoch": 2.240060015003751,
1172
+ "step": 1120
1173
+ },
1174
+ {
1175
+ "loss": 0.01608244627714157,
1176
+ "grad_norm": 0.2359917163848877,
1177
+ "learning_rate": 0.00018357165706774767,
1178
+ "entropy": 0.021389624777657445,
1179
+ "num_tokens": 7160997.0,
1180
+ "mean_token_accuracy": 0.9940837919712067,
1181
+ "epoch": 2.2600650162540634,
1182
+ "step": 1130
1183
+ },
1184
+ {
1185
+ "loss": 0.020376379787921905,
1186
+ "grad_norm": 0.07118914276361465,
1187
+ "learning_rate": 0.00018320662340704609,
1188
+ "entropy": 0.02022790874907514,
1189
+ "num_tokens": 7226007.0,
1190
+ "mean_token_accuracy": 0.9942706093192101,
1191
+ "epoch": 2.280070017504376,
1192
+ "step": 1140
1193
+ },
1194
+ {
1195
+ "loss": 0.01612715721130371,
1196
+ "grad_norm": 0.25310513377189636,
1197
+ "learning_rate": 0.00018283795001908457,
1198
+ "entropy": 0.01867675751855131,
1199
+ "num_tokens": 7287986.0,
1200
+ "mean_token_accuracy": 0.9942055746912957,
1201
+ "epoch": 2.3000750187546886,
1202
+ "step": 1150
1203
+ },
1204
+ {
1205
+ "loss": 0.016605789959430694,
1206
+ "grad_norm": 0.2142266184091568,
1207
+ "learning_rate": 0.0001824656530308315,
1208
+ "entropy": 0.015577676898101345,
1209
+ "num_tokens": 7349421.0,
1210
+ "mean_token_accuracy": 0.9948085315525532,
1211
+ "epoch": 2.320080020005001,
1212
+ "step": 1160
1213
+ },
1214
+ {
1215
+ "loss": 0.018196111917495726,
1216
+ "grad_norm": 0.07947535812854767,
1217
+ "learning_rate": 0.00018208974872776322,
1218
+ "entropy": 0.020357475349737798,
1219
+ "num_tokens": 7411941.0,
1220
+ "mean_token_accuracy": 0.993843074887991,
1221
+ "epoch": 2.3400850212553137,
1222
+ "step": 1170
1223
+ },
1224
+ {
1225
+ "loss": 0.016091108322143555,
1226
+ "grad_norm": 0.19202570617198944,
1227
+ "learning_rate": 0.00018171025355315164,
1228
+ "entropy": 0.017023067966511006,
1229
+ "num_tokens": 7474356.0,
1230
+ "mean_token_accuracy": 0.9953217662870883,
1231
+ "epoch": 2.3600900225056263,
1232
+ "step": 1180
1233
+ },
1234
+ {
1235
+ "loss": 0.015425822138786316,
1236
+ "grad_norm": 0.24460658431053162,
1237
+ "learning_rate": 0.00018132718410734515,
1238
+ "entropy": 0.01825423450791277,
1239
+ "num_tokens": 7536565.0,
1240
+ "mean_token_accuracy": 0.9943965286016464,
1241
+ "epoch": 2.380095023755939,
1242
+ "step": 1190
1243
+ },
1244
+ {
1245
+ "loss": 0.013514925539493561,
1246
+ "grad_norm": 0.24659694731235504,
1247
+ "learning_rate": 0.00018094055714704225,
1248
+ "entropy": 0.015542891589575447,
1249
+ "num_tokens": 7599751.0,
1250
+ "mean_token_accuracy": 0.9943179704248906,
1251
+ "epoch": 2.4001000250062514,
1252
+ "step": 1200
1253
+ },
1254
+ {
1255
+ "loss": 0.012726837396621704,
1256
+ "grad_norm": 0.2550601065158844,
1257
+ "learning_rate": 0.0001805503895845587,
1258
+ "entropy": 0.015307287167524919,
1259
+ "num_tokens": 7661520.0,
1260
+ "mean_token_accuracy": 0.9962130591273308,
1261
+ "epoch": 2.420105026256564,
1262
+ "step": 1210
1263
+ },
1264
+ {
1265
+ "loss": 0.012257835268974305,
1266
+ "grad_norm": 0.15603283047676086,
1267
+ "learning_rate": 0.00018015669848708767,
1268
+ "entropy": 0.01749844834121177,
1269
+ "num_tokens": 7726291.0,
1270
+ "mean_token_accuracy": 0.9954387851059436,
1271
+ "epoch": 2.4401100275068766,
1272
+ "step": 1220
1273
+ },
1274
+ {
1275
+ "loss": 0.018561355769634247,
1276
+ "grad_norm": 0.22746174037456512,
1277
+ "learning_rate": 0.0001797595010759531,
1278
+ "entropy": 0.019737370508664753,
1279
+ "num_tokens": 7788766.0,
1280
+ "mean_token_accuracy": 0.9940820440649987,
1281
+ "epoch": 2.460115028757189,
1282
+ "step": 1230
1283
+ },
1284
+ {
1285
+ "loss": 0.013849316537380219,
1286
+ "grad_norm": 0.3152976334095001,
1287
+ "learning_rate": 0.0001793588147258565,
1288
+ "entropy": 0.015929855390277227,
1289
+ "num_tokens": 7851396.0,
1290
+ "mean_token_accuracy": 0.9952766291797162,
1291
+ "epoch": 2.4801200300075017,
1292
+ "step": 1240
1293
+ },
1294
+ {
1295
+ "loss": 0.019299986958503722,
1296
+ "grad_norm": 0.2762889862060547,
1297
+ "learning_rate": 0.00017895465696411692,
1298
+ "entropy": 0.02108022033935413,
1299
+ "num_tokens": 7915421.0,
1300
+ "mean_token_accuracy": 0.9940553769469261,
1301
+ "epoch": 2.5001250312578147,
1302
+ "step": 1250
1303
+ },
1304
+ {
1305
+ "loss": 0.020875005424022673,
1306
+ "grad_norm": 0.24086585640907288,
1307
+ "learning_rate": 0.00017854704546990408,
1308
+ "entropy": 0.020866505106096157,
1309
+ "num_tokens": 7977133.0,
1310
+ "mean_token_accuracy": 0.9940896175801754,
1311
+ "epoch": 2.520130032508127,
1312
+ "step": 1260
1313
+ },
1314
+ {
1315
+ "loss": 0.018643879890441896,
1316
+ "grad_norm": 0.23781460523605347,
1317
+ "learning_rate": 0.0001781359980734653,
1318
+ "entropy": 0.020857046739547514,
1319
+ "num_tokens": 8040191.0,
1320
+ "mean_token_accuracy": 0.9929048053920269,
1321
+ "epoch": 2.54013503375844,
1322
+ "step": 1270
1323
+ },
1324
+ {
1325
+ "loss": 0.017947974801063537,
1326
+ "grad_norm": 0.2054099142551422,
1327
+ "learning_rate": 0.0001777215327553452,
1328
+ "entropy": 0.021198420476866885,
1329
+ "num_tokens": 8100152.0,
1330
+ "mean_token_accuracy": 0.9934561550617218,
1331
+ "epoch": 2.560140035008752,
1332
+ "step": 1280
1333
+ },
1334
+ {
1335
+ "loss": 0.0151192307472229,
1336
+ "grad_norm": 0.469458669424057,
1337
+ "learning_rate": 0.00017730366764559955,
1338
+ "entropy": 0.018285114454920405,
1339
+ "num_tokens": 8162591.0,
1340
+ "mean_token_accuracy": 0.9951836079359054,
1341
+ "epoch": 2.580145036259065,
1342
+ "step": 1290
1343
+ },
1344
+ {
1345
+ "loss": 0.0184975802898407,
1346
+ "grad_norm": 0.2882782518863678,
1347
+ "learning_rate": 0.00017688242102300192,
1348
+ "entropy": 0.018374070005665998,
1349
+ "num_tokens": 8225224.0,
1350
+ "mean_token_accuracy": 0.9948060251772404,
1351
+ "epoch": 2.600150037509377,
1352
+ "step": 1300
1353
+ },
1354
+ {
1355
+ "loss": 0.023296315968036652,
1356
+ "grad_norm": 0.21924524009227753,
1357
+ "learning_rate": 0.00017645781131424423,
1358
+ "entropy": 0.022974171601526906,
1359
+ "num_tokens": 8290274.0,
1360
+ "mean_token_accuracy": 0.9924322210252285,
1361
+ "epoch": 2.62015503875969,
1362
+ "step": 1310
1363
+ },
1364
+ {
1365
+ "loss": 0.017917373776435853,
1366
+ "grad_norm": 0.354758620262146,
1367
+ "learning_rate": 0.00017602985709313073,
1368
+ "entropy": 0.02169415617827326,
1369
+ "num_tokens": 8354370.0,
1370
+ "mean_token_accuracy": 0.9948078036308289,
1371
+ "epoch": 2.6401600400100023,
1372
+ "step": 1320
1373
+ },
1374
+ {
1375
+ "loss": 0.012286465615034103,
1376
+ "grad_norm": 0.4151551127433777,
1377
+ "learning_rate": 0.00017559857707976536,
1378
+ "entropy": 0.016268294051405972,
1379
+ "num_tokens": 8415022.0,
1380
+ "mean_token_accuracy": 0.9961770802736283,
1381
+ "epoch": 2.6601650412603153,
1382
+ "step": 1330
1383
+ },
1384
+ {
1385
+ "loss": 0.01499750316143036,
1386
+ "grad_norm": 0.4065402150154114,
1387
+ "learning_rate": 0.0001751639901397331,
1388
+ "entropy": 0.0170176492218161,
1389
+ "num_tokens": 8478286.0,
1390
+ "mean_token_accuracy": 0.9952280893921852,
1391
+ "epoch": 2.6801700425106274,
1392
+ "step": 1340
1393
+ },
1394
+ {
1395
+ "loss": 0.015018537640571594,
1396
+ "grad_norm": 0.1335880309343338,
1397
+ "learning_rate": 0.0001747261152832746,
1398
+ "entropy": 0.018367627350380646,
1399
+ "num_tokens": 8541154.0,
1400
+ "mean_token_accuracy": 0.9949840374290944,
1401
+ "epoch": 2.7001750437609404,
1402
+ "step": 1350
1403
+ },
1404
+ {
1405
+ "loss": 0.019237272441387177,
1406
+ "grad_norm": 0.20553363859653473,
1407
+ "learning_rate": 0.00017428497166445452,
1408
+ "entropy": 0.019403737914399245,
1409
+ "num_tokens": 8605361.0,
1410
+ "mean_token_accuracy": 0.9930156201124192,
1411
+ "epoch": 2.7201800450112525,
1412
+ "step": 1360
1413
+ },
1414
+ {
1415
+ "loss": 0.019147740304470064,
1416
+ "grad_norm": 0.31576329469680786,
1417
+ "learning_rate": 0.00017384057858032393,
1418
+ "entropy": 0.02298831292137038,
1419
+ "num_tokens": 8669155.0,
1420
+ "mean_token_accuracy": 0.9930574476718903,
1421
+ "epoch": 2.7401850462615656,
1422
+ "step": 1370
1423
+ },
1424
+ {
1425
+ "loss": 0.017152118682861327,
1426
+ "grad_norm": 0.21389739215373993,
1427
+ "learning_rate": 0.00017339295547007594,
1428
+ "entropy": 0.01718737747578416,
1429
+ "num_tokens": 8735019.0,
1430
+ "mean_token_accuracy": 0.995334691554308,
1431
+ "epoch": 2.760190047511878,
1432
+ "step": 1380
1433
+ },
1434
+ {
1435
+ "loss": 0.019429606199264527,
1436
+ "grad_norm": 0.26539650559425354,
1437
+ "learning_rate": 0.00017294212191419547,
1438
+ "entropy": 0.019683032816101332,
1439
+ "num_tokens": 8800972.0,
1440
+ "mean_token_accuracy": 0.9936951123178005,
1441
+ "epoch": 2.7801950487621907,
1442
+ "step": 1390
1443
+ },
1444
+ {
1445
+ "loss": 0.01909356415271759,
1446
+ "grad_norm": 0.2084941565990448,
1447
+ "learning_rate": 0.00017248809763360277,
1448
+ "entropy": 0.018636453218641692,
1449
+ "num_tokens": 8867596.0,
1450
+ "mean_token_accuracy": 0.9940625011920929,
1451
+ "epoch": 2.8002000500125033,
1452
+ "step": 1400
1453
+ },
1454
+ {
1455
+ "loss": 0.018046848475933075,
1456
+ "grad_norm": 0.21403200924396515,
1457
+ "learning_rate": 0.0001720309024887907,
1458
+ "entropy": 0.023316194582730532,
1459
+ "num_tokens": 8927516.0,
1460
+ "mean_token_accuracy": 0.9939217306673527,
1461
+ "epoch": 2.820205051262816,
1462
+ "step": 1410
1463
+ },
1464
+ {
1465
+ "loss": 0.018755699694156646,
1466
+ "grad_norm": 0.33148443698883057,
1467
+ "learning_rate": 0.000171570556478956,
1468
+ "entropy": 0.01655098560877377,
1469
+ "num_tokens": 8989089.0,
1470
+ "mean_token_accuracy": 0.9942230053246022,
1471
+ "epoch": 2.8402100525131284,
1472
+ "step": 1420
1473
+ },
1474
+ {
1475
+ "loss": 0.024141687154769897,
1476
+ "grad_norm": 0.18521511554718018,
1477
+ "learning_rate": 0.00017110707974112447,
1478
+ "entropy": 0.02549898542056326,
1479
+ "num_tokens": 9054435.0,
1480
+ "mean_token_accuracy": 0.9930150359869003,
1481
+ "epoch": 2.860215053763441,
1482
+ "step": 1430
1483
+ },
1484
+ {
1485
+ "loss": 0.020157690346240997,
1486
+ "grad_norm": 0.23067928850650787,
1487
+ "learning_rate": 0.0001706404925492701,
1488
+ "entropy": 0.01900827525241766,
1489
+ "num_tokens": 9114274.0,
1490
+ "mean_token_accuracy": 0.9943146407604218,
1491
+ "epoch": 2.8802200550137536,
1492
+ "step": 1440
1493
+ },
1494
+ {
1495
+ "loss": 0.014147150516510009,
1496
+ "grad_norm": 0.5235961079597473,
1497
+ "learning_rate": 0.00017017081531342813,
1498
+ "entropy": 0.01623811650206335,
1499
+ "num_tokens": 9174841.0,
1500
+ "mean_token_accuracy": 0.9944271765649318,
1501
+ "epoch": 2.900225056264066,
1502
+ "step": 1450
1503
+ },
1504
+ {
1505
+ "loss": 0.022856634855270386,
1506
+ "grad_norm": 0.11377973109483719,
1507
+ "learning_rate": 0.00016969806857880241,
1508
+ "entropy": 0.023971330239146483,
1509
+ "num_tokens": 9239456.0,
1510
+ "mean_token_accuracy": 0.9932261377573013,
1511
+ "epoch": 2.9202300575143787,
1512
+ "step": 1460
1513
+ },
1514
+ {
1515
+ "loss": 0.02486345320940018,
1516
+ "grad_norm": 0.12835904955863953,
1517
+ "learning_rate": 0.00016922227302486667,
1518
+ "entropy": 0.02552748184389202,
1519
+ "num_tokens": 9305144.0,
1520
+ "mean_token_accuracy": 0.9918816141784191,
1521
+ "epoch": 2.9402350587646913,
1522
+ "step": 1470
1523
+ },
1524
+ {
1525
+ "loss": 0.018220885097980498,
1526
+ "grad_norm": 0.18033206462860107,
1527
+ "learning_rate": 0.00016874344946445974,
1528
+ "entropy": 0.019867337332107125,
1529
+ "num_tokens": 9365854.0,
1530
+ "mean_token_accuracy": 0.9945706635713577,
1531
+ "epoch": 2.960240060015004,
1532
+ "step": 1480
1533
+ },
1534
+ {
1535
+ "loss": 0.01618766337633133,
1536
+ "grad_norm": 0.2495020180940628,
1537
+ "learning_rate": 0.00016826161884287533,
1538
+ "entropy": 0.01948691344150575,
1539
+ "num_tokens": 9427287.0,
1540
+ "mean_token_accuracy": 0.9956672258675099,
1541
+ "epoch": 2.9802450612653164,
1542
+ "step": 1490
1543
+ },
1544
+ {
1545
+ "loss": 0.024902991950511932,
1546
+ "grad_norm": 0.2727943956851959,
1547
+ "learning_rate": 0.00016777680223694575,
1548
+ "entropy": 0.022937397798228586,
1549
+ "num_tokens": 9488514.0,
1550
+ "mean_token_accuracy": 0.9909723401069641,
1551
+ "epoch": 3.0,
1552
+ "step": 1500
1553
+ },
1554
+ {
1555
+ "eval_loss": 0.03645886108279228,
1556
+ "eval_runtime": 69.2921,
1557
+ "eval_samples_per_second": 14.937,
1558
+ "eval_steps_per_second": 7.476,
1559
+ "eval_entropy": 0.02839457441272365,
1560
+ "eval_num_tokens": 9488514.0,
1561
+ "eval_mean_token_accuracy": 0.9902446437986661,
1562
+ "epoch": 3.0,
1563
+ "step": 1500
1564
+ },
1565
+ {
1566
+ "loss": 0.014892478287220002,
1567
+ "grad_norm": 0.26308536529541016,
1568
+ "learning_rate": 0.00016728902085411996,
1569
+ "entropy": 0.02168392370658694,
1570
+ "num_tokens": 9553582.0,
1571
+ "mean_token_accuracy": 0.9945793129503727,
1572
+ "epoch": 3.0200050012503126,
1573
+ "step": 1510
1574
+ },
1575
+ {
1576
+ "loss": 0.009397410601377488,
1577
+ "grad_norm": 0.20036503672599792,
1578
+ "learning_rate": 0.0001667982960315358,
1579
+ "entropy": 0.012075830744288396,
1580
+ "num_tokens": 9617281.0,
1581
+ "mean_token_accuracy": 0.9964840039610863,
1582
+ "epoch": 3.040010002500625,
1583
+ "step": 1520
1584
+ },
1585
+ {
1586
+ "loss": 0.015942367911338805,
1587
+ "grad_norm": 0.3861319124698639,
1588
+ "learning_rate": 0.00016630464923508677,
1589
+ "entropy": 0.014315767139487434,
1590
+ "num_tokens": 9681345.0,
1591
+ "mean_token_accuracy": 0.9952807635068893,
1592
+ "epoch": 3.0600150037509377,
1593
+ "step": 1530
1594
+ },
1595
+ {
1596
+ "loss": 0.01565181165933609,
1597
+ "grad_norm": 0.23598027229309082,
1598
+ "learning_rate": 0.00016580810205848288,
1599
+ "entropy": 0.01963768747918948,
1600
+ "num_tokens": 9743616.0,
1601
+ "mean_token_accuracy": 0.9945429727435112,
1602
+ "epoch": 3.0800200050012503,
1603
+ "step": 1540
1604
+ },
1605
+ {
1606
+ "loss": 0.01511429101228714,
1607
+ "grad_norm": 0.3226635754108429,
1608
+ "learning_rate": 0.0001653086762223063,
1609
+ "entropy": 0.01306593646404508,
1610
+ "num_tokens": 9808581.0,
1611
+ "mean_token_accuracy": 0.995248269289732,
1612
+ "epoch": 3.100025006251563,
1613
+ "step": 1550
1614
+ },
1615
+ {
1616
+ "loss": 0.009530831128358841,
1617
+ "grad_norm": 0.09154416620731354,
1618
+ "learning_rate": 0.00016480639357306098,
1619
+ "entropy": 0.013261715146654751,
1620
+ "num_tokens": 9873666.0,
1621
+ "mean_token_accuracy": 0.9966560050845146,
1622
+ "epoch": 3.1200300075018754,
1623
+ "step": 1560
1624
+ },
1625
+ {
1626
+ "loss": 0.009393466264009475,
1627
+ "grad_norm": 0.21701541543006897,
1628
+ "learning_rate": 0.00016430127608221714,
1629
+ "entropy": 0.011822419746022206,
1630
+ "num_tokens": 9937438.0,
1631
+ "mean_token_accuracy": 0.9964691713452339,
1632
+ "epoch": 3.140035008752188,
1633
+ "step": 1570
1634
+ },
1635
+ {
1636
+ "loss": 0.011047683656215668,
1637
+ "grad_norm": 0.2678857743740082,
1638
+ "learning_rate": 0.00016379334584525025,
1639
+ "entropy": 0.012122366849507672,
1640
+ "num_tokens": 9999440.0,
1641
+ "mean_token_accuracy": 0.9967051848769188,
1642
+ "epoch": 3.1600400100025006,
1643
+ "step": 1580
1644
+ },
1645
+ {
1646
+ "loss": 0.013723762333393097,
1647
+ "grad_norm": 0.324455201625824,
1648
+ "learning_rate": 0.00016328262508067431,
1649
+ "entropy": 0.012699224287644029,
1650
+ "num_tokens": 10061600.0,
1651
+ "mean_token_accuracy": 0.9955959998071193,
1652
+ "epoch": 3.180045011252813,
1653
+ "step": 1590
1654
+ },
1655
+ {
1656
+ "loss": 0.014421728253364564,
1657
+ "grad_norm": 0.18887697160243988,
1658
+ "learning_rate": 0.00016276913612907007,
1659
+ "entropy": 0.020439925385289825,
1660
+ "num_tokens": 10126520.0,
1661
+ "mean_token_accuracy": 0.9959282651543617,
1662
+ "epoch": 3.2000500125031257,
1663
+ "step": 1600
1664
+ },
1665
+ {
1666
+ "loss": 0.01239979937672615,
1667
+ "grad_norm": 0.1129460483789444,
1668
+ "learning_rate": 0.00016225290145210772,
1669
+ "entropy": 0.012199809608864598,
1670
+ "num_tokens": 10191553.0,
1671
+ "mean_token_accuracy": 0.995978644490242,
1672
+ "epoch": 3.2200550137534383,
1673
+ "step": 1610
1674
+ },
1675
+ {
1676
+ "loss": 0.014834728837013245,
1677
+ "grad_norm": 0.20427951216697693,
1678
+ "learning_rate": 0.00016173394363156444,
1679
+ "entropy": 0.016009513070457615,
1680
+ "num_tokens": 10258289.0,
1681
+ "mean_token_accuracy": 0.9960135422647,
1682
+ "epoch": 3.240060015003751,
1683
+ "step": 1620
1684
+ },
1685
+ {
1686
+ "loss": 0.014402203261852264,
1687
+ "grad_norm": 0.25807589292526245,
1688
+ "learning_rate": 0.00016121228536833645,
1689
+ "entropy": 0.015858568061958067,
1690
+ "num_tokens": 10324281.0,
1691
+ "mean_token_accuracy": 0.9963105775415897,
1692
+ "epoch": 3.2600650162540634,
1693
+ "step": 1630
1694
+ },
1695
+ {
1696
+ "loss": 0.014248587191104889,
1697
+ "grad_norm": 0.13609230518341064,
1698
+ "learning_rate": 0.00016068794948144617,
1699
+ "entropy": 0.015872705554647835,
1700
+ "num_tokens": 10385122.0,
1701
+ "mean_token_accuracy": 0.9952699325978756,
1702
+ "epoch": 3.280070017504376,
1703
+ "step": 1640
1704
+ },
1705
+ {
1706
+ "loss": 0.013415993750095367,
1707
+ "grad_norm": 0.21917292475700378,
1708
+ "learning_rate": 0.00016016095890704387,
1709
+ "entropy": 0.010807368888345081,
1710
+ "num_tokens": 10447821.0,
1711
+ "mean_token_accuracy": 0.9959428884088993,
1712
+ "epoch": 3.3000750187546886,
1713
+ "step": 1650
1714
+ },
1715
+ {
1716
+ "loss": 0.008242987841367722,
1717
+ "grad_norm": 0.2698808014392853,
1718
+ "learning_rate": 0.0001596313366974045,
1719
+ "entropy": 0.012187929065839853,
1720
+ "num_tokens": 10507896.0,
1721
+ "mean_token_accuracy": 0.9972235180437565,
1722
+ "epoch": 3.320080020005001,
1723
+ "step": 1660
1724
+ },
1725
+ {
1726
+ "loss": 0.013128173351287842,
1727
+ "grad_norm": 0.20612332224845886,
1728
+ "learning_rate": 0.00015909910601991922,
1729
+ "entropy": 0.013906099726591492,
1730
+ "num_tokens": 10569117.0,
1731
+ "mean_token_accuracy": 0.9961497314274311,
1732
+ "epoch": 3.3400850212553137,
1733
+ "step": 1670
1734
+ },
1735
+ {
1736
+ "loss": 0.01361977458000183,
1737
+ "grad_norm": 0.13489088416099548,
1738
+ "learning_rate": 0.00015856429015608209,
1739
+ "entropy": 0.013951514207292348,
1740
+ "num_tokens": 10630981.0,
1741
+ "mean_token_accuracy": 0.9956404089927673,
1742
+ "epoch": 3.3600900225056263,
1743
+ "step": 1680
1744
+ },
1745
+ {
1746
+ "loss": 0.014680840075016022,
1747
+ "grad_norm": 0.2713385820388794,
1748
+ "learning_rate": 0.00015802691250047153,
1749
+ "entropy": 0.013167628296650946,
1750
+ "num_tokens": 10693173.0,
1751
+ "mean_token_accuracy": 0.9950266376137733,
1752
+ "epoch": 3.380095023755939,
1753
+ "step": 1690
1754
+ },
1755
+ {
1756
+ "loss": 0.013400137424468994,
1757
+ "grad_norm": 0.1426415741443634,
1758
+ "learning_rate": 0.00015748699655972708,
1759
+ "entropy": 0.017778589528461453,
1760
+ "num_tokens": 10753429.0,
1761
+ "mean_token_accuracy": 0.9946111224591732,
1762
+ "epoch": 3.4001000250062514,
1763
+ "step": 1700
1764
+ },
1765
+ {
1766
+ "loss": 0.008509327471256257,
1767
+ "grad_norm": 0.16599993407726288,
1768
+ "learning_rate": 0.00015694456595152106,
1769
+ "entropy": 0.013743974023964257,
1770
+ "num_tokens": 10819863.0,
1771
+ "mean_token_accuracy": 0.9970920436084271,
1772
+ "epoch": 3.420105026256564,
1773
+ "step": 1710
1774
+ },
1775
+ {
1776
+ "loss": 0.01316998302936554,
1777
+ "grad_norm": 0.20159605145454407,
1778
+ "learning_rate": 0.0001563996444035255,
1779
+ "entropy": 0.01153940933654667,
1780
+ "num_tokens": 10882060.0,
1781
+ "mean_token_accuracy": 0.9955882236361504,
1782
+ "epoch": 3.4401100275068766,
1783
+ "step": 1720
1784
+ },
1785
+ {
1786
+ "loss": 0.011991073936223983,
1787
+ "grad_norm": 0.23580631613731384,
1788
+ "learning_rate": 0.00015585225575237427,
1789
+ "entropy": 0.012680305907269939,
1790
+ "num_tokens": 10945979.0,
1791
+ "mean_token_accuracy": 0.9958546876907348,
1792
+ "epoch": 3.460115028757189,
1793
+ "step": 1730
1794
+ },
1795
+ {
1796
+ "loss": 0.012041158974170685,
1797
+ "grad_norm": 0.33789604902267456,
1798
+ "learning_rate": 0.00015530242394262017,
1799
+ "entropy": 0.013596625554055209,
1800
+ "num_tokens": 11008355.0,
1801
+ "mean_token_accuracy": 0.9955416478216648,
1802
+ "epoch": 3.4801200300075017,
1803
+ "step": 1740
1804
+ },
1805
+ {
1806
+ "loss": 0.01627572178840637,
1807
+ "grad_norm": 0.2994728982448578,
1808
+ "learning_rate": 0.00015475017302568782,
1809
+ "entropy": 0.019999047268356662,
1810
+ "num_tokens": 11071934.0,
1811
+ "mean_token_accuracy": 0.9931987822055817,
1812
+ "epoch": 3.5001250312578147,
1813
+ "step": 1750
1814
+ },
1815
+ {
1816
+ "loss": 0.011943073570728302,
1817
+ "grad_norm": 0.09841946512460709,
1818
+ "learning_rate": 0.00015419552715882138,
1819
+ "entropy": 0.01364866496878676,
1820
+ "num_tokens": 11132456.0,
1821
+ "mean_token_accuracy": 0.9967477336525917,
1822
+ "epoch": 3.520130032508127,
1823
+ "step": 1760
1824
+ },
1825
+ {
1826
+ "loss": 0.0153175488114357,
1827
+ "grad_norm": 0.22312118113040924,
1828
+ "learning_rate": 0.00015363851060402783,
1829
+ "entropy": 0.016219895507674664,
1830
+ "num_tokens": 11194782.0,
1831
+ "mean_token_accuracy": 0.9945069424808025,
1832
+ "epoch": 3.54013503375844,
1833
+ "step": 1770
1834
+ },
1835
+ {
1836
+ "loss": 0.00787808895111084,
1837
+ "grad_norm": 0.16057588160037994,
1838
+ "learning_rate": 0.0001530791477270158,
1839
+ "entropy": 0.012033771253481974,
1840
+ "num_tokens": 11259307.0,
1841
+ "mean_token_accuracy": 0.9970344088971614,
1842
+ "epoch": 3.560140035008752,
1843
+ "step": 1780
1844
+ },
1845
+ {
1846
+ "loss": 0.013948053121566772,
1847
+ "grad_norm": 0.2074371874332428,
1848
+ "learning_rate": 0.0001525174629961296,
1849
+ "entropy": 0.01175229620130267,
1850
+ "num_tokens": 11321733.0,
1851
+ "mean_token_accuracy": 0.9959047585725784,
1852
+ "epoch": 3.580145036259065,
1853
+ "step": 1790
1854
+ },
1855
+ {
1856
+ "loss": 0.012327873706817627,
1857
+ "grad_norm": 0.15074662864208221,
1858
+ "learning_rate": 0.00015195348098127895,
1859
+ "entropy": 0.014628350256680278,
1860
+ "num_tokens": 11384586.0,
1861
+ "mean_token_accuracy": 0.9950341537594796,
1862
+ "epoch": 3.600150037509377,
1863
+ "step": 1800
1864
+ },
1865
+ {
1866
+ "loss": 0.012766703963279724,
1867
+ "grad_norm": 0.2124103605747223,
1868
+ "learning_rate": 0.00015138722635286422,
1869
+ "entropy": 0.018070634140167387,
1870
+ "num_tokens": 11448156.0,
1871
+ "mean_token_accuracy": 0.9958022043108941,
1872
+ "epoch": 3.62015503875969,
1873
+ "step": 1810
1874
+ },
1875
+ {
1876
+ "loss": 0.011698020249605178,
1877
+ "grad_norm": 0.4769149124622345,
1878
+ "learning_rate": 0.0001508187238806973,
1879
+ "entropy": 0.01308420468485565,
1880
+ "num_tokens": 11511882.0,
1881
+ "mean_token_accuracy": 0.9961448684334755,
1882
+ "epoch": 3.6401600400100023,
1883
+ "step": 1820
1884
+ },
1885
+ {
1886
+ "loss": 0.014116832613945007,
1887
+ "grad_norm": 0.25340843200683594,
1888
+ "learning_rate": 0.00015024799843291802,
1889
+ "entropy": 0.012517862502500065,
1890
+ "num_tokens": 11575526.0,
1891
+ "mean_token_accuracy": 0.9950296327471733,
1892
+ "epoch": 3.6601650412603153,
1893
+ "step": 1830
1894
+ },
1895
+ {
1896
+ "loss": 0.014804676175117493,
1897
+ "grad_norm": 0.21997588872909546,
1898
+ "learning_rate": 0.00014967507497490635,
1899
+ "entropy": 0.016702812965377234,
1900
+ "num_tokens": 11637521.0,
1901
+ "mean_token_accuracy": 0.9941891603171825,
1902
+ "epoch": 3.6801700425106274,
1903
+ "step": 1840
1904
+ },
1905
+ {
1906
+ "loss": 0.008088278025388718,
1907
+ "grad_norm": 0.1356344372034073,
1908
+ "learning_rate": 0.00014909997856819032,
1909
+ "entropy": 0.01380894306494156,
1910
+ "num_tokens": 11700080.0,
1911
+ "mean_token_accuracy": 0.9972025558352471,
1912
+ "epoch": 3.7001750437609404,
1913
+ "step": 1850
1914
+ },
1915
+ {
1916
+ "loss": 0.014773441851139069,
1917
+ "grad_norm": 0.2926776707172394,
1918
+ "learning_rate": 0.00014852273436934986,
1919
+ "entropy": 0.013712721945921658,
1920
+ "num_tokens": 11761687.0,
1921
+ "mean_token_accuracy": 0.9959305942058563,
1922
+ "epoch": 3.7201800450112525,
1923
+ "step": 1860
1924
+ },
1925
+ {
1926
+ "loss": 0.011582046002149581,
1927
+ "grad_norm": 0.13980428874492645,
1928
+ "learning_rate": 0.00014794336762891623,
1929
+ "entropy": 0.014661396172596142,
1930
+ "num_tokens": 11823454.0,
1931
+ "mean_token_accuracy": 0.9961862593889237,
1932
+ "epoch": 3.7401850462615656,
1933
+ "step": 1870
1934
+ },
1935
+ {
1936
+ "loss": 0.012114252895116806,
1937
+ "grad_norm": 0.2284947633743286,
1938
+ "learning_rate": 0.00014736190369026754,
1939
+ "entropy": 0.01451311390119372,
1940
+ "num_tokens": 11884267.0,
1941
+ "mean_token_accuracy": 0.9961981892585754,
1942
+ "epoch": 3.760190047511878,
1943
+ "step": 1880
1944
+ },
1945
+ {
1946
+ "loss": 0.007772183418273926,
1947
+ "grad_norm": 0.255247563123703,
1948
+ "learning_rate": 0.0001467783679885201,
1949
+ "entropy": 0.010285129089606927,
1950
+ "num_tokens": 11948880.0,
1951
+ "mean_token_accuracy": 0.9978563249111175,
1952
+ "epoch": 3.7801950487621907,
1953
+ "step": 1890
1954
+ },
1955
+ {
1956
+ "loss": 0.013219112157821655,
1957
+ "grad_norm": 0.21494613587856293,
1958
+ "learning_rate": 0.00014619278604941603,
1959
+ "entropy": 0.01082497325114673,
1960
+ "num_tokens": 12012547.0,
1961
+ "mean_token_accuracy": 0.9958216808736324,
1962
+ "epoch": 3.8002000500125033,
1963
+ "step": 1900
1964
+ },
1965
+ {
1966
+ "loss": 0.012468833476305008,
1967
+ "grad_norm": 0.28526443243026733,
1968
+ "learning_rate": 0.00014560518348820625,
1969
+ "entropy": 0.013233061737264507,
1970
+ "num_tokens": 12078372.0,
1971
+ "mean_token_accuracy": 0.9958592697978019,
1972
+ "epoch": 3.820205051262816,
1973
+ "step": 1910
1974
+ },
1975
+ {
1976
+ "loss": 0.018085935711860658,
1977
+ "grad_norm": 0.1471295952796936,
1978
+ "learning_rate": 0.00014501558600853035,
1979
+ "entropy": 0.015974047601775964,
1980
+ "num_tokens": 12142500.0,
1981
+ "mean_token_accuracy": 0.9947570398449898,
1982
+ "epoch": 3.8402100525131284,
1983
+ "step": 1920
1984
+ },
1985
+ {
1986
+ "loss": 0.013111139833927154,
1987
+ "grad_norm": 0.2544702887535095,
1988
+ "learning_rate": 0.0001444240194012922,
1989
+ "entropy": 0.01969735559250694,
1990
+ "num_tokens": 12206205.0,
1991
+ "mean_token_accuracy": 0.9956923462450504,
1992
+ "epoch": 3.860215053763441,
1993
+ "step": 1930
1994
+ },
1995
+ {
1996
+ "loss": 0.013215355575084686,
1997
+ "grad_norm": 0.14971140027046204,
1998
+ "learning_rate": 0.00014383050954353154,
1999
+ "entropy": 0.01610187725018477,
2000
+ "num_tokens": 12269828.0,
2001
+ "mean_token_accuracy": 0.9956071071326733,
2002
+ "epoch": 3.8802200550137536,
2003
+ "step": 1940
2004
+ },
2005
+ {
2006
+ "loss": 0.00939919427037239,
2007
+ "grad_norm": 0.08307329565286636,
2008
+ "learning_rate": 0.00014323508239729232,
2009
+ "entropy": 0.01351477519783657,
2010
+ "num_tokens": 12331919.0,
2011
+ "mean_token_accuracy": 0.99660724401474,
2012
+ "epoch": 3.900225056264066,
2013
+ "step": 1950
2014
+ },
2015
+ {
2016
+ "loss": 0.011213938146829605,
2017
+ "grad_norm": 0.12744051218032837,
2018
+ "learning_rate": 0.00014263776400848678,
2019
+ "entropy": 0.011691810854972572,
2020
+ "num_tokens": 12394410.0,
2021
+ "mean_token_accuracy": 0.9957682631909848,
2022
+ "epoch": 3.9202300575143787,
2023
+ "step": 1960
2024
+ },
2025
+ {
2026
+ "loss": 0.014179202914237975,
2027
+ "grad_norm": 0.30811241269111633,
2028
+ "learning_rate": 0.00014203858050575632,
2029
+ "entropy": 0.013546516641508789,
2030
+ "num_tokens": 12458473.0,
2031
+ "mean_token_accuracy": 0.9955927409231663,
2032
+ "epoch": 3.9402350587646913,
2033
+ "step": 1970
2034
+ },
2035
+ {
2036
+ "loss": 0.014471597969532013,
2037
+ "grad_norm": 0.1450197547674179,
2038
+ "learning_rate": 0.00014143755809932845,
2039
+ "entropy": 0.016499465111701285,
2040
+ "num_tokens": 12523348.0,
2041
+ "mean_token_accuracy": 0.9954880520701408,
2042
+ "epoch": 3.960240060015004,
2043
+ "step": 1980
2044
+ },
2045
+ {
2046
+ "loss": 0.010644648969173432,
2047
+ "grad_norm": 0.14227765798568726,
2048
+ "learning_rate": 0.0001408347230798702,
2049
+ "entropy": 0.013392421180469682,
2050
+ "num_tokens": 12588443.0,
2051
+ "mean_token_accuracy": 0.9969804167747498,
2052
+ "epoch": 3.9802450612653164,
2053
+ "step": 1990
2054
+ },
2055
+ {
2056
+ "loss": 0.015882152318954467,
2057
+ "grad_norm": 0.12858586013317108,
2058
+ "learning_rate": 0.00014023010181733826,
2059
+ "entropy": 0.01455020939060369,
2060
+ "num_tokens": 12651352.0,
2061
+ "mean_token_accuracy": 0.9946769113782086,
2062
+ "epoch": 4.0,
2063
+ "step": 2000
2064
+ },
2065
+ {
2066
+ "eval_loss": 0.0426219180226326,
2067
+ "eval_runtime": 69.3664,
2068
+ "eval_samples_per_second": 14.921,
2069
+ "eval_steps_per_second": 7.468,
2070
+ "eval_entropy": 0.03254893434532418,
2071
+ "eval_num_tokens": 12651352.0,
2072
+ "eval_mean_token_accuracy": 0.9884336234059573,
2073
+ "epoch": 4.0,
2074
+ "step": 2000
2075
+ },
2076
+ {
2077
+ "loss": 0.00863734856247902,
2078
+ "grad_norm": 0.4523090422153473,
2079
+ "learning_rate": 0.00013962372075982543,
2080
+ "entropy": 0.01400298816661234,
2081
+ "num_tokens": 12710379.0,
2082
+ "mean_token_accuracy": 0.9968750610947609,
2083
+ "epoch": 4.020005001250313,
2084
+ "step": 2010
2085
+ },
2086
+ {
2087
+ "loss": 0.007897177338600158,
2088
+ "grad_norm": 0.10824855417013168,
2089
+ "learning_rate": 0.0001390156064324035,
2090
+ "entropy": 0.008752083206127281,
2091
+ "num_tokens": 12774117.0,
2092
+ "mean_token_accuracy": 0.9971247158944607,
2093
+ "epoch": 4.040010002500625,
2094
+ "step": 2020
2095
+ },
2096
+ {
2097
+ "loss": 0.008237957209348678,
2098
+ "grad_norm": 0.3647105097770691,
2099
+ "learning_rate": 0.00013840578543596315,
2100
+ "entropy": 0.008517850490170531,
2101
+ "num_tokens": 12838463.0,
2102
+ "mean_token_accuracy": 0.9974852904677391,
2103
+ "epoch": 4.060015003750938,
2104
+ "step": 2030
2105
+ },
2106
+ {
2107
+ "loss": 0.010357823967933655,
2108
+ "grad_norm": 0.1849185675382614,
2109
+ "learning_rate": 0.00013779428444605035,
2110
+ "entropy": 0.011237980193618569,
2111
+ "num_tokens": 12906031.0,
2112
+ "mean_token_accuracy": 0.9965578347444535,
2113
+ "epoch": 4.08002000500125,
2114
+ "step": 2040
2115
+ },
2116
+ {
2117
+ "loss": 0.00867396593093872,
2118
+ "grad_norm": 0.17338383197784424,
2119
+ "learning_rate": 0.0001371811302116994,
2120
+ "entropy": 0.00913453484463389,
2121
+ "num_tokens": 12968016.0,
2122
+ "mean_token_accuracy": 0.9973611943423748,
2123
+ "epoch": 4.100025006251563,
2124
+ "step": 2050
2125
+ },
2126
+ {
2127
+ "loss": 0.012893503904342652,
2128
+ "grad_norm": 0.1852918565273285,
2129
+ "learning_rate": 0.0001365663495542628,
2130
+ "entropy": 0.012978466165077408,
2131
+ "num_tokens": 13032518.0,
2132
+ "mean_token_accuracy": 0.99602395221591,
2133
+ "epoch": 4.120030007501875,
2134
+ "step": 2060
2135
+ },
2136
+ {
2137
+ "loss": 0.010699793696403503,
2138
+ "grad_norm": 0.2881818115711212,
2139
+ "learning_rate": 0.00013594996936623814,
2140
+ "entropy": 0.015379714348819106,
2141
+ "num_tokens": 13096744.0,
2142
+ "mean_token_accuracy": 0.9955209918320179,
2143
+ "epoch": 4.140035008752188,
2144
+ "step": 2070
2145
+ },
2146
+ {
2147
+ "loss": 0.009091087430715562,
2148
+ "grad_norm": 0.32742437720298767,
2149
+ "learning_rate": 0.0001353320166100917,
2150
+ "entropy": 0.009939568623667582,
2151
+ "num_tokens": 13162095.0,
2152
+ "mean_token_accuracy": 0.9969943076372146,
2153
+ "epoch": 4.160040010002501,
2154
+ "step": 2080
2155
+ },
2156
+ {
2157
+ "loss": 0.009758947789669037,
2158
+ "grad_norm": 0.2090802937746048,
2159
+ "learning_rate": 0.00013471251831707884,
2160
+ "entropy": 0.010461670262520784,
2161
+ "num_tokens": 13228156.0,
2162
+ "mean_token_accuracy": 0.9968024276196956,
2163
+ "epoch": 4.180045011252814,
2164
+ "step": 2090
2165
+ },
2166
+ {
2167
+ "loss": 0.008040308952331543,
2168
+ "grad_norm": 0.21466973423957825,
2169
+ "learning_rate": 0.0001340915015860618,
2170
+ "entropy": 0.011725931792898336,
2171
+ "num_tokens": 13291512.0,
2172
+ "mean_token_accuracy": 0.9968237906694413,
2173
+ "epoch": 4.200050012503126,
2174
+ "step": 2100
2175
+ },
2176
+ {
2177
+ "loss": 0.00803126096725464,
2178
+ "grad_norm": 0.24069944024085999,
2179
+ "learning_rate": 0.0001334689935823243,
2180
+ "entropy": 0.011084824410500006,
2181
+ "num_tokens": 13353896.0,
2182
+ "mean_token_accuracy": 0.997739101946354,
2183
+ "epoch": 4.220055013753439,
2184
+ "step": 2110
2185
+ },
2186
+ {
2187
+ "loss": 0.008842091262340545,
2188
+ "grad_norm": 0.06226912513375282,
2189
+ "learning_rate": 0.00013284502153638295,
2190
+ "entropy": 0.010992687013640534,
2191
+ "num_tokens": 13416743.0,
2192
+ "mean_token_accuracy": 0.996946869045496,
2193
+ "epoch": 4.240060015003751,
2194
+ "step": 2120
2195
+ },
2196
+ {
2197
+ "loss": 0.010905887186527252,
2198
+ "grad_norm": 0.11721836030483246,
2199
+ "learning_rate": 0.00013221961274279654,
2200
+ "entropy": 0.011237628920207498,
2201
+ "num_tokens": 13481284.0,
2202
+ "mean_token_accuracy": 0.9970996268093586,
2203
+ "epoch": 4.260065016254064,
2204
+ "step": 2130
2205
+ },
2206
+ {
2207
+ "loss": 0.012466417998075486,
2208
+ "grad_norm": 0.3521440327167511,
2209
+ "learning_rate": 0.00013159279455897166,
2210
+ "entropy": 0.010562418565677944,
2211
+ "num_tokens": 13543678.0,
2212
+ "mean_token_accuracy": 0.9958751887083054,
2213
+ "epoch": 4.280070017504376,
2214
+ "step": 2140
2215
+ },
2216
+ {
2217
+ "loss": 0.011747314780950546,
2218
+ "grad_norm": 0.26293885707855225,
2219
+ "learning_rate": 0.0001309645944039663,
2220
+ "entropy": 0.014598401125113014,
2221
+ "num_tokens": 13608112.0,
2222
+ "mean_token_accuracy": 0.9962641790509223,
2223
+ "epoch": 4.300075018754689,
2224
+ "step": 2150
2225
+ },
2226
+ {
2227
+ "loss": 0.007845200598239899,
2228
+ "grad_norm": 0.16723529994487762,
2229
+ "learning_rate": 0.00013033503975729035,
2230
+ "entropy": 0.011412217889301246,
2231
+ "num_tokens": 13667979.0,
2232
+ "mean_token_accuracy": 0.99744006767869,
2233
+ "epoch": 4.320080020005001,
2234
+ "step": 2160
2235
+ },
2236
+ {
2237
+ "loss": 0.010894352942705155,
2238
+ "grad_norm": 0.21131186187267303,
2239
+ "learning_rate": 0.0001297041581577035,
2240
+ "entropy": 0.010380906579666772,
2241
+ "num_tokens": 13730034.0,
2242
+ "mean_token_accuracy": 0.9965024016797542,
2243
+ "epoch": 4.340085021255314,
2244
+ "step": 2170
2245
+ },
2246
+ {
2247
+ "loss": 0.00956823006272316,
2248
+ "grad_norm": 0.10813307017087936,
2249
+ "learning_rate": 0.00012907197720201071,
2250
+ "entropy": 0.009887772376168868,
2251
+ "num_tokens": 13792366.0,
2252
+ "mean_token_accuracy": 0.9968511998653412,
2253
+ "epoch": 4.360090022505626,
2254
+ "step": 2180
2255
+ },
2256
+ {
2257
+ "loss": 0.009695110470056533,
2258
+ "grad_norm": 0.21164672076702118,
2259
+ "learning_rate": 0.00012843852454385497,
2260
+ "entropy": 0.011902359123632777,
2261
+ "num_tokens": 13853048.0,
2262
+ "mean_token_accuracy": 0.9974331922829152,
2263
+ "epoch": 4.380095023755939,
2264
+ "step": 2190
2265
+ },
2266
+ {
2267
+ "loss": 0.008793818950653075,
2268
+ "grad_norm": 0.4802285432815552,
2269
+ "learning_rate": 0.00012780382789250762,
2270
+ "entropy": 0.009543133205443154,
2271
+ "num_tokens": 13917884.0,
2272
+ "mean_token_accuracy": 0.997763866186142,
2273
+ "epoch": 4.400100025006251,
2274
+ "step": 2200
2275
+ },
2276
+ {
2277
+ "loss": 0.009752951562404633,
2278
+ "grad_norm": 0.3032193183898926,
2279
+ "learning_rate": 0.00012716791501165634,
2280
+ "entropy": 0.01140449561207788,
2281
+ "num_tokens": 13982842.0,
2282
+ "mean_token_accuracy": 0.9972191534936428,
2283
+ "epoch": 4.420105026256564,
2284
+ "step": 2210
2285
+ },
2286
+ {
2287
+ "loss": 0.005666728690266609,
2288
+ "grad_norm": 0.394113153219223,
2289
+ "learning_rate": 0.00012653081371819064,
2290
+ "entropy": 0.008245287769386777,
2291
+ "num_tokens": 14045445.0,
2292
+ "mean_token_accuracy": 0.998167161643505,
2293
+ "epoch": 4.4401100275068766,
2294
+ "step": 2220
2295
+ },
2296
+ {
2297
+ "loss": 0.010289490967988969,
2298
+ "grad_norm": 0.5735944509506226,
2299
+ "learning_rate": 0.00012589255188098498,
2300
+ "entropy": 0.008848439441499068,
2301
+ "num_tokens": 14105270.0,
2302
+ "mean_token_accuracy": 0.996202427148819,
2303
+ "epoch": 4.46011502875719,
2304
+ "step": 2230
2305
+ },
2306
+ {
2307
+ "loss": 0.006267648935317993,
2308
+ "grad_norm": 0.0710088387131691,
2309
+ "learning_rate": 0.00012525315741967978,
2310
+ "entropy": 0.010472280244721332,
2311
+ "num_tokens": 14166955.0,
2312
+ "mean_token_accuracy": 0.9972212843596935,
2313
+ "epoch": 4.480120030007502,
2314
+ "step": 2240
2315
+ },
2316
+ {
2317
+ "loss": 0.010307309031486512,
2318
+ "grad_norm": 0.23172006011009216,
2319
+ "learning_rate": 0.00012461265830346018,
2320
+ "entropy": 0.008165620337967994,
2321
+ "num_tokens": 14231397.0,
2322
+ "mean_token_accuracy": 0.996221523731947,
2323
+ "epoch": 4.500125031257815,
2324
+ "step": 2250
2325
+ },
2326
+ {
2327
+ "loss": 0.010486914217472077,
2328
+ "grad_norm": 0.1562025099992752,
2329
+ "learning_rate": 0.00012397108254983243,
2330
+ "entropy": 0.01140070731707965,
2331
+ "num_tokens": 14293502.0,
2332
+ "mean_token_accuracy": 0.9965781837701797,
2333
+ "epoch": 4.520130032508127,
2334
+ "step": 2260
2335
+ },
2336
+ {
2337
+ "loss": 0.008283475786447525,
2338
+ "grad_norm": 0.09583538770675659,
2339
+ "learning_rate": 0.00012332845822339846,
2340
+ "entropy": 0.011921767683816142,
2341
+ "num_tokens": 14355038.0,
2342
+ "mean_token_accuracy": 0.9969584576785564,
2343
+ "epoch": 4.54013503375844,
2344
+ "step": 2270
2345
+ },
2346
+ {
2347
+ "loss": 0.00834668129682541,
2348
+ "grad_norm": 0.18446850776672363,
2349
+ "learning_rate": 0.000122684813434628,
2350
+ "entropy": 0.010264168232242809,
2351
+ "num_tokens": 14420148.0,
2352
+ "mean_token_accuracy": 0.9971672594547272,
2353
+ "epoch": 4.560140035008752,
2354
+ "step": 2280
2355
+ },
2356
+ {
2357
+ "loss": 0.009413036704063415,
2358
+ "grad_norm": 0.11777894198894501,
2359
+ "learning_rate": 0.00012204017633862932,
2360
+ "entropy": 0.011230875192995881,
2361
+ "num_tokens": 14482015.0,
2362
+ "mean_token_accuracy": 0.9962861262261867,
2363
+ "epoch": 4.580145036259065,
2364
+ "step": 2290
2365
+ },
2366
+ {
2367
+ "loss": 0.0066015787422657015,
2368
+ "grad_norm": 0.14621251821517944,
2369
+ "learning_rate": 0.00012139457513391732,
2370
+ "entropy": 0.007982138471561484,
2371
+ "num_tokens": 14546515.0,
2372
+ "mean_token_accuracy": 0.9983268916606903,
2373
+ "epoch": 4.600150037509377,
2374
+ "step": 2300
2375
+ },
2376
+ {
2377
+ "loss": 0.010532976686954498,
2378
+ "grad_norm": 0.2948937714099884,
2379
+ "learning_rate": 0.00012074803806118016,
2380
+ "entropy": 0.011228621992995614,
2381
+ "num_tokens": 14610218.0,
2382
+ "mean_token_accuracy": 0.9960615806281566,
2383
+ "epoch": 4.62015503875969,
2384
+ "step": 2310
2385
+ },
2386
+ {
2387
+ "loss": 0.012350869178771973,
2388
+ "grad_norm": 0.25452813506126404,
2389
+ "learning_rate": 0.0001201005934020439,
2390
+ "entropy": 0.0075275591014360545,
2391
+ "num_tokens": 14673523.0,
2392
+ "mean_token_accuracy": 0.9973488107323647,
2393
+ "epoch": 4.640160040010002,
2394
+ "step": 2320
2395
+ },
2396
+ {
2397
+ "loss": 0.015287771821022034,
2398
+ "grad_norm": 0.24262821674346924,
2399
+ "learning_rate": 0.00011945226947783532,
2400
+ "entropy": 0.012186212290544063,
2401
+ "num_tokens": 14734854.0,
2402
+ "mean_token_accuracy": 0.995884881913662,
2403
+ "epoch": 4.660165041260315,
2404
+ "step": 2330
2405
+ },
2406
+ {
2407
+ "loss": 0.010069895535707474,
2408
+ "grad_norm": 0.24241459369659424,
2409
+ "learning_rate": 0.00011880309464834317,
2410
+ "entropy": 0.014627664355066372,
2411
+ "num_tokens": 14796717.0,
2412
+ "mean_token_accuracy": 0.9970944046974182,
2413
+ "epoch": 4.680170042510627,
2414
+ "step": 2340
2415
+ },
2416
+ {
2417
+ "loss": 0.007354290783405304,
2418
+ "grad_norm": 0.24272984266281128,
2419
+ "learning_rate": 0.00011815309731057752,
2420
+ "entropy": 0.00997849061677698,
2421
+ "num_tokens": 14857558.0,
2422
+ "mean_token_accuracy": 0.9976357862353324,
2423
+ "epoch": 4.70017504376094,
2424
+ "step": 2350
2425
+ },
2426
+ {
2427
+ "loss": 0.01080884262919426,
2428
+ "grad_norm": 0.16180723905563354,
2429
+ "learning_rate": 0.00011750230589752762,
2430
+ "entropy": 0.011729113181354478,
2431
+ "num_tokens": 14918129.0,
2432
+ "mean_token_accuracy": 0.9959106191992759,
2433
+ "epoch": 4.7201800450112525,
2434
+ "step": 2360
2435
+ },
2436
+ {
2437
+ "loss": 0.0070034988224506375,
2438
+ "grad_norm": 0.14300037920475006,
2439
+ "learning_rate": 0.00011685074887691815,
2440
+ "entropy": 0.010010966495974572,
2441
+ "num_tokens": 14981742.0,
2442
+ "mean_token_accuracy": 0.9970420770347118,
2443
+ "epoch": 4.740185046261566,
2444
+ "step": 2370
2445
+ },
2446
+ {
2447
+ "loss": 0.006338100135326386,
2448
+ "grad_norm": 0.31339094042778015,
2449
+ "learning_rate": 0.00011619845474996396,
2450
+ "entropy": 0.007486376102315262,
2451
+ "num_tokens": 15044749.0,
2452
+ "mean_token_accuracy": 0.998369050770998,
2453
+ "epoch": 4.760190047511878,
2454
+ "step": 2380
2455
+ },
2456
+ {
2457
+ "loss": 0.0088754341006279,
2458
+ "grad_norm": 0.2821660339832306,
2459
+ "learning_rate": 0.0001155454520501233,
2460
+ "entropy": 0.0094100816793798,
2461
+ "num_tokens": 15107356.0,
2462
+ "mean_token_accuracy": 0.9967978030443192,
2463
+ "epoch": 4.780195048762191,
2464
+ "step": 2390
2465
+ },
2466
+ {
2467
+ "loss": 0.0074894212186336516,
2468
+ "grad_norm": 0.20098550617694855,
2469
+ "learning_rate": 0.00011489176934184966,
2470
+ "entropy": 0.011058530519949272,
2471
+ "num_tokens": 15170161.0,
2472
+ "mean_token_accuracy": 0.9970949165523052,
2473
+ "epoch": 4.800200050012503,
2474
+ "step": 2400
2475
+ },
2476
+ {
2477
+ "loss": 0.006931519508361817,
2478
+ "grad_norm": 0.2597576379776001,
2479
+ "learning_rate": 0.00011423743521934239,
2480
+ "entropy": 0.009024496493293555,
2481
+ "num_tokens": 15233166.0,
2482
+ "mean_token_accuracy": 0.9981255434453488,
2483
+ "epoch": 4.820205051262816,
2484
+ "step": 2410
2485
+ },
2486
+ {
2487
+ "loss": 0.007713411748409271,
2488
+ "grad_norm": 0.07438832521438599,
2489
+ "learning_rate": 0.00011358247830529572,
2490
+ "entropy": 0.008874563317294814,
2491
+ "num_tokens": 15294745.0,
2492
+ "mean_token_accuracy": 0.997540271282196,
2493
+ "epoch": 4.840210052513128,
2494
+ "step": 2420
2495
+ },
2496
+ {
2497
+ "loss": 0.010046067833900451,
2498
+ "grad_norm": 0.15564411878585815,
2499
+ "learning_rate": 0.00011292692724964684,
2500
+ "entropy": 0.01154620652741869,
2501
+ "num_tokens": 15360934.0,
2502
+ "mean_token_accuracy": 0.9968956887722016,
2503
+ "epoch": 4.860215053763441,
2504
+ "step": 2430
2505
+ },
2506
+ {
2507
+ "loss": 0.007103031873703003,
2508
+ "grad_norm": 0.13080066442489624,
2509
+ "learning_rate": 0.00011227081072832264,
2510
+ "entropy": 0.009648629953153432,
2511
+ "num_tokens": 15423505.0,
2512
+ "mean_token_accuracy": 0.9975046671926975,
2513
+ "epoch": 4.880220055013753,
2514
+ "step": 2440
2515
+ },
2516
+ {
2517
+ "loss": 0.011635245382785797,
2518
+ "grad_norm": 0.1020200327038765,
2519
+ "learning_rate": 0.00011161415744198529,
2520
+ "entropy": 0.01059294661572494,
2521
+ "num_tokens": 15488354.0,
2522
+ "mean_token_accuracy": 0.996422378718853,
2523
+ "epoch": 4.900225056264066,
2524
+ "step": 2450
2525
+ },
2526
+ {
2527
+ "loss": 0.00895947739481926,
2528
+ "grad_norm": 0.5128690600395203,
2529
+ "learning_rate": 0.00011095699611477672,
2530
+ "entropy": 0.01126530086476123,
2531
+ "num_tokens": 15551047.0,
2532
+ "mean_token_accuracy": 0.9972493521869182,
2533
+ "epoch": 4.920230057514378,
2534
+ "step": 2460
2535
+ },
2536
+ {
2537
+ "loss": 0.007447101175785065,
2538
+ "grad_norm": 0.1531531810760498,
2539
+ "learning_rate": 0.00011029935549306236,
2540
+ "entropy": 0.009639624001283664,
2541
+ "num_tokens": 15614133.0,
2542
+ "mean_token_accuracy": 0.9972159087657928,
2543
+ "epoch": 4.940235058764691,
2544
+ "step": 2470
2545
+ },
2546
+ {
2547
+ "loss": 0.009513139724731445,
2548
+ "grad_norm": 0.109550341963768,
2549
+ "learning_rate": 0.00010964126434417348,
2550
+ "entropy": 0.011513816034130287,
2551
+ "num_tokens": 15681338.0,
2552
+ "mean_token_accuracy": 0.9967059269547462,
2553
+ "epoch": 4.960240060015003,
2554
+ "step": 2480
2555
+ },
2556
+ {
2557
+ "loss": 0.014146287739276887,
2558
+ "grad_norm": 0.2227025330066681,
2559
+ "learning_rate": 0.00010898275145514878,
2560
+ "entropy": 0.012698486276713083,
2561
+ "num_tokens": 15750237.0,
2562
+ "mean_token_accuracy": 0.9964324481785297,
2563
+ "epoch": 4.980245061265316,
2564
+ "step": 2490
2565
+ },
2566
+ {
2567
+ "loss": 0.011340580135583877,
2568
+ "grad_norm": 0.26418933272361755,
2569
+ "learning_rate": 0.00010832384563147539,
2570
+ "entropy": 0.012120508169607979,
2571
+ "num_tokens": 15814190.0,
2572
+ "mean_token_accuracy": 0.9962437522562244,
2573
+ "epoch": 5.0,
2574
+ "step": 2500
2575
+ },
2576
+ {
2577
+ "eval_loss": 0.036649439483881,
2578
+ "eval_runtime": 68.931,
2579
+ "eval_samples_per_second": 15.015,
2580
+ "eval_steps_per_second": 7.515,
2581
+ "eval_entropy": 0.01951373018991628,
2582
+ "eval_num_tokens": 15814190.0,
2583
+ "eval_mean_token_accuracy": 0.990988754052453,
2584
+ "epoch": 5.0,
2585
+ "step": 2500
2586
+ },
2587
+ {
2588
+ "loss": 0.00565112940967083,
2589
+ "grad_norm": 0.24155987799167633,
2590
+ "learning_rate": 0.00010766457569582855,
2591
+ "entropy": 0.008389830897795037,
2592
+ "num_tokens": 15876304.0,
2593
+ "mean_token_accuracy": 0.9987288475036621,
2594
+ "epoch": 5.020005001250313,
2595
+ "step": 2510
2596
+ },
2597
+ {
2598
+ "loss": 0.004657933115959167,
2599
+ "grad_norm": 0.17384299635887146,
2600
+ "learning_rate": 0.000107004970486811,
2601
+ "entropy": 0.0063808045721089005,
2602
+ "num_tokens": 15939328.0,
2603
+ "mean_token_accuracy": 0.9986095182597637,
2604
+ "epoch": 5.040010002500625,
2605
+ "step": 2520
2606
+ },
2607
+ {
2608
+ "loss": 0.009442869573831558,
2609
+ "grad_norm": 0.07942040264606476,
2610
+ "learning_rate": 0.00010634505885769134,
2611
+ "entropy": 0.008581503613459062,
2612
+ "num_tokens": 16001585.0,
2613
+ "mean_token_accuracy": 0.9973239652812481,
2614
+ "epoch": 5.060015003750938,
2615
+ "step": 2530
2616
+ },
2617
+ {
2618
+ "loss": 0.007105500251054764,
2619
+ "grad_norm": 0.1442539244890213,
2620
+ "learning_rate": 0.00010568486967514218,
2621
+ "entropy": 0.008049002460029441,
2622
+ "num_tokens": 16066250.0,
2623
+ "mean_token_accuracy": 0.9976261213421822,
2624
+ "epoch": 5.08002000500125,
2625
+ "step": 2540
2626
+ },
2627
+ {
2628
+ "loss": 0.006882892549037933,
2629
+ "grad_norm": 0.05469922348856926,
2630
+ "learning_rate": 0.00010502443181797697,
2631
+ "entropy": 0.008640473794366698,
2632
+ "num_tokens": 16131511.0,
2633
+ "mean_token_accuracy": 0.9982132039964199,
2634
+ "epoch": 5.100025006251563,
2635
+ "step": 2550
2636
+ },
2637
+ {
2638
+ "loss": 0.007951027154922486,
2639
+ "grad_norm": 0.08848944306373596,
2640
+ "learning_rate": 0.00010436377417588714,
2641
+ "entropy": 0.007696686122289975,
2642
+ "num_tokens": 16194361.0,
2643
+ "mean_token_accuracy": 0.998307142406702,
2644
+ "epoch": 5.120030007501875,
2645
+ "step": 2560
2646
+ },
2647
+ {
2648
+ "loss": 0.005469654873013497,
2649
+ "grad_norm": 0.1663186103105545,
2650
+ "learning_rate": 0.0001037029256481782,
2651
+ "entropy": 0.008428349181485828,
2652
+ "num_tokens": 16257949.0,
2653
+ "mean_token_accuracy": 0.9982100896537304,
2654
+ "epoch": 5.140035008752188,
2655
+ "step": 2570
2656
+ },
2657
+ {
2658
+ "loss": 0.007561860978603363,
2659
+ "grad_norm": 0.1089382916688919,
2660
+ "learning_rate": 0.00010304191514250559,
2661
+ "entropy": 0.00860111919027986,
2662
+ "num_tokens": 16319368.0,
2663
+ "mean_token_accuracy": 0.9977700248360634,
2664
+ "epoch": 5.160040010002501,
2665
+ "step": 2580
2666
+ },
2667
+ {
2668
+ "loss": 0.006244239956140518,
2669
+ "grad_norm": 0.29137757420539856,
2670
+ "learning_rate": 0.00010238077157361021,
2671
+ "entropy": 0.006166888629377354,
2672
+ "num_tokens": 16382222.0,
2673
+ "mean_token_accuracy": 0.9981474541127682,
2674
+ "epoch": 5.180045011252814,
2675
+ "step": 2590
2676
+ },
2677
+ {
2678
+ "loss": 0.004823794960975647,
2679
+ "grad_norm": 0.09055764973163605,
2680
+ "learning_rate": 0.00010171952386205354,
2681
+ "entropy": 0.007182681497215526,
2682
+ "num_tokens": 16442713.0,
2683
+ "mean_token_accuracy": 0.9988246329128743,
2684
+ "epoch": 5.200050012503126,
2685
+ "step": 2600
2686
+ },
2687
+ {
2688
+ "loss": 0.004676086083054543,
2689
+ "grad_norm": 0.041668012738227844,
2690
+ "learning_rate": 0.00010105820093295268,
2691
+ "entropy": 0.007433413605031092,
2692
+ "num_tokens": 16507336.0,
2693
+ "mean_token_accuracy": 0.9984147787094116,
2694
+ "epoch": 5.220055013753439,
2695
+ "step": 2610
2696
+ },
2697
+ {
2698
+ "loss": 0.007142329961061478,
2699
+ "grad_norm": 0.3844437897205353,
2700
+ "learning_rate": 0.00010039683171471493,
2701
+ "entropy": 0.006547862391016679,
2702
+ "num_tokens": 16573852.0,
2703
+ "mean_token_accuracy": 0.9983086295425891,
2704
+ "epoch": 5.240060015003751,
2705
+ "step": 2620
2706
+ },
2707
+ {
2708
+ "loss": 0.0057931594550609585,
2709
+ "grad_norm": 0.12350230664014816,
2710
+ "learning_rate": 9.973544513777245e-05,
2711
+ "entropy": 0.007999910078433458,
2712
+ "num_tokens": 16636798.0,
2713
+ "mean_token_accuracy": 0.998402901738882,
2714
+ "epoch": 5.260065016254064,
2715
+ "step": 2630
2716
+ },
2717
+ {
2718
+ "loss": 0.006187746301293373,
2719
+ "grad_norm": 0.10715912282466888,
2720
+ "learning_rate": 9.907407013331663e-05,
2721
+ "entropy": 0.007076963333383901,
2722
+ "num_tokens": 16702698.0,
2723
+ "mean_token_accuracy": 0.9976529404520988,
2724
+ "epoch": 5.280070017504376,
2725
+ "step": 2640
2726
+ },
2727
+ {
2728
+ "loss": 0.004311095550656318,
2729
+ "grad_norm": 0.3015357255935669,
2730
+ "learning_rate": 9.841273563203277e-05,
2731
+ "entropy": 0.005494680403353413,
2732
+ "num_tokens": 16765527.0,
2733
+ "mean_token_accuracy": 0.9985795192420482,
2734
+ "epoch": 5.300075018754689,
2735
+ "step": 2650
2736
+ },
2737
+ {
2738
+ "loss": 0.005706357955932617,
2739
+ "grad_norm": 0.06844376027584076,
2740
+ "learning_rate": 9.77514705628344e-05,
2741
+ "entropy": 0.008315026501077228,
2742
+ "num_tokens": 16828006.0,
2743
+ "mean_token_accuracy": 0.9986971832811833,
2744
+ "epoch": 5.320080020005001,
2745
+ "step": 2660
2746
+ },
2747
+ {
2748
+ "loss": 0.00420297309756279,
2749
+ "grad_norm": 0.15605595707893372,
2750
+ "learning_rate": 9.709030385159785e-05,
2751
+ "entropy": 0.007199513514206046,
2752
+ "num_tokens": 16891023.0,
2753
+ "mean_token_accuracy": 0.9988421566784382,
2754
+ "epoch": 5.340085021255314,
2755
+ "step": 2670
2756
+ },
2757
+ {
2758
+ "loss": 0.00714726448059082,
2759
+ "grad_norm": 0.19374419748783112,
2760
+ "learning_rate": 9.642926441989696e-05,
2761
+ "entropy": 0.005838291899635806,
2762
+ "num_tokens": 16956668.0,
2763
+ "mean_token_accuracy": 0.9979994632303715,
2764
+ "epoch": 5.360090022505626,
2765
+ "step": 2680
2766
+ },
2767
+ {
2768
+ "loss": 0.005109471827745437,
2769
+ "grad_norm": 0.1971459686756134,
2770
+ "learning_rate": 9.57683811837379e-05,
2771
+ "entropy": 0.009764070269011427,
2772
+ "num_tokens": 17020153.0,
2773
+ "mean_token_accuracy": 0.9985188752412796,
2774
+ "epoch": 5.380095023755939,
2775
+ "step": 2690
2776
+ },
2777
+ {
2778
+ "loss": 0.006213615462183952,
2779
+ "grad_norm": 0.2851213216781616,
2780
+ "learning_rate": 9.510768305229441e-05,
2781
+ "entropy": 0.008533979691856075,
2782
+ "num_tokens": 17084832.0,
2783
+ "mean_token_accuracy": 0.9977529622614384,
2784
+ "epoch": 5.400100025006251,
2785
+ "step": 2700
2786
+ },
2787
+ {
2788
+ "loss": 0.0061513490974903105,
2789
+ "grad_norm": 0.10104754567146301,
2790
+ "learning_rate": 9.444719892664311e-05,
2791
+ "entropy": 0.007165478614842869,
2792
+ "num_tokens": 17149790.0,
2793
+ "mean_token_accuracy": 0.9975666105747223,
2794
+ "epoch": 5.420105026256564,
2795
+ "step": 2710
2796
+ },
2797
+ {
2798
+ "loss": 0.005662607401609421,
2799
+ "grad_norm": 0.09378691017627716,
2800
+ "learning_rate": 9.378695769849931e-05,
2801
+ "entropy": 0.007089314765835297,
2802
+ "num_tokens": 17213202.0,
2803
+ "mean_token_accuracy": 0.9981832534074784,
2804
+ "epoch": 5.4401100275068766,
2805
+ "step": 2720
2806
+ },
2807
+ {
2808
+ "loss": 0.006653358787298202,
2809
+ "grad_norm": 0.12766185402870178,
2810
+ "learning_rate": 9.312698824895328e-05,
2811
+ "entropy": 0.006666196065634722,
2812
+ "num_tokens": 17278955.0,
2813
+ "mean_token_accuracy": 0.9976298168301583,
2814
+ "epoch": 5.46011502875719,
2815
+ "step": 2730
2816
+ },
2817
+ {
2818
+ "loss": 0.006521254032850266,
2819
+ "grad_norm": 0.02992381900548935,
2820
+ "learning_rate": 9.246731944720675e-05,
2821
+ "entropy": 0.007897612718807068,
2822
+ "num_tokens": 17344058.0,
2823
+ "mean_token_accuracy": 0.9974993944168091,
2824
+ "epoch": 5.480120030007502,
2825
+ "step": 2740
2826
+ },
2827
+ {
2828
+ "loss": 0.0059999067336320875,
2829
+ "grad_norm": 0.1317594200372696,
2830
+ "learning_rate": 9.180798014931006e-05,
2831
+ "entropy": 0.006773643911219551,
2832
+ "num_tokens": 17404192.0,
2833
+ "mean_token_accuracy": 0.9978866256773472,
2834
+ "epoch": 5.500125031257815,
2835
+ "step": 2750
2836
+ },
2837
+ {
2838
+ "loss": 0.00612783133983612,
2839
+ "grad_norm": 0.2199675589799881,
2840
+ "learning_rate": 9.11489991969001e-05,
2841
+ "entropy": 0.00654012646737101,
2842
+ "num_tokens": 17467965.0,
2843
+ "mean_token_accuracy": 0.9980414494872093,
2844
+ "epoch": 5.520130032508127,
2845
+ "step": 2760
2846
+ },
2847
+ {
2848
+ "loss": 0.007611159235239029,
2849
+ "grad_norm": 0.15056708455085754,
2850
+ "learning_rate": 9.049040541593854e-05,
2851
+ "entropy": 0.006149375334643992,
2852
+ "num_tokens": 17528827.0,
2853
+ "mean_token_accuracy": 0.9979180261492729,
2854
+ "epoch": 5.54013503375844,
2855
+ "step": 2770
2856
+ },
2857
+ {
2858
+ "loss": 0.007264629751443863,
2859
+ "grad_norm": 0.1319355070590973,
2860
+ "learning_rate": 8.98322276154509e-05,
2861
+ "entropy": 0.008627775539207506,
2862
+ "num_tokens": 17589448.0,
2863
+ "mean_token_accuracy": 0.9976452924311161,
2864
+ "epoch": 5.560140035008752,
2865
+ "step": 2780
2866
+ },
2867
+ {
2868
+ "loss": 0.003955654054880142,
2869
+ "grad_norm": 0.20958350598812103,
2870
+ "learning_rate": 8.917449458626636e-05,
2871
+ "entropy": 0.005277934976766119,
2872
+ "num_tokens": 17652540.0,
2873
+ "mean_token_accuracy": 0.9986450515687466,
2874
+ "epoch": 5.580145036259065,
2875
+ "step": 2790
2876
+ },
2877
+ {
2878
+ "loss": 0.004750333726406097,
2879
+ "grad_norm": 0.14845848083496094,
2880
+ "learning_rate": 8.85172350997584e-05,
2881
+ "entropy": 0.005947213277613628,
2882
+ "num_tokens": 17719311.0,
2883
+ "mean_token_accuracy": 0.9983809232711792,
2884
+ "epoch": 5.600150037509377,
2885
+ "step": 2800
2886
+ },
2887
+ {
2888
+ "loss": 0.006387320905923843,
2889
+ "grad_norm": 0.025019152089953423,
2890
+ "learning_rate": 8.786047790658619e-05,
2891
+ "entropy": 0.006668693362371414,
2892
+ "num_tokens": 17783740.0,
2893
+ "mean_token_accuracy": 0.9980740129947663,
2894
+ "epoch": 5.62015503875969,
2895
+ "step": 2810
2896
+ },
2897
+ {
2898
+ "loss": 0.003687459230422974,
2899
+ "grad_norm": 0.12704399228096008,
2900
+ "learning_rate": 8.720425173543694e-05,
2901
+ "entropy": 0.006505226148146903,
2902
+ "num_tokens": 17848561.0,
2903
+ "mean_token_accuracy": 0.9982961259782315,
2904
+ "epoch": 5.640160040010002,
2905
+ "step": 2820
2906
+ },
2907
+ {
2908
+ "loss": 0.004726226255297661,
2909
+ "grad_norm": 0.35164615511894226,
2910
+ "learning_rate": 8.654858529176926e-05,
2911
+ "entropy": 0.006050794995826436,
2912
+ "num_tokens": 17911026.0,
2913
+ "mean_token_accuracy": 0.9985501445829869,
2914
+ "epoch": 5.660165041260315,
2915
+ "step": 2830
2916
+ },
2917
+ {
2918
+ "loss": 0.007286908477544785,
2919
+ "grad_norm": 0.10387425869703293,
2920
+ "learning_rate": 8.589350725655745e-05,
2921
+ "entropy": 0.006026100756207598,
2922
+ "num_tokens": 17973473.0,
2923
+ "mean_token_accuracy": 0.9979210302233696,
2924
+ "epoch": 5.680170042510627,
2925
+ "step": 2840
2926
+ },
2927
+ {
2928
+ "loss": 0.0033504638820886614,
2929
+ "grad_norm": 0.26233962178230286,
2930
+ "learning_rate": 8.523904628503695e-05,
2931
+ "entropy": 0.005752308326191269,
2932
+ "num_tokens": 18037116.0,
2933
+ "mean_token_accuracy": 0.9988096967339516,
2934
+ "epoch": 5.70017504376094,
2935
+ "step": 2850
2936
+ },
2937
+ {
2938
+ "loss": 0.005085382983088494,
2939
+ "grad_norm": 0.10810191929340363,
2940
+ "learning_rate": 8.458523100545077e-05,
2941
+ "entropy": 0.006968487899939646,
2942
+ "num_tokens": 18102687.0,
2943
+ "mean_token_accuracy": 0.99852888956666,
2944
+ "epoch": 5.7201800450112525,
2945
+ "step": 2860
2946
+ },
2947
+ {
2948
+ "loss": 0.005646209418773651,
2949
+ "grad_norm": 0.14457252621650696,
2950
+ "learning_rate": 8.393209001779736e-05,
2951
+ "entropy": 0.007342068195430329,
2952
+ "num_tokens": 18165237.0,
2953
+ "mean_token_accuracy": 0.9982465572655201,
2954
+ "epoch": 5.740185046261566,
2955
+ "step": 2870
2956
+ },
2957
+ {
2958
+ "loss": 0.004496878013014793,
2959
+ "grad_norm": 0.2686709463596344,
2960
+ "learning_rate": 8.327965189257937e-05,
2961
+ "entropy": 0.006458067610947182,
2962
+ "num_tokens": 18224314.0,
2963
+ "mean_token_accuracy": 0.9987071558833123,
2964
+ "epoch": 5.760190047511878,
2965
+ "step": 2880
2966
+ },
2967
+ {
2968
+ "loss": 0.006712660938501358,
2969
+ "grad_norm": 0.08530271798372269,
2970
+ "learning_rate": 8.262794516955404e-05,
2971
+ "entropy": 0.005851120350780547,
2972
+ "num_tokens": 18289806.0,
2973
+ "mean_token_accuracy": 0.9982900738716125,
2974
+ "epoch": 5.780195048762191,
2975
+ "step": 2890
2976
+ },
2977
+ {
2978
+ "loss": 0.005763013660907745,
2979
+ "grad_norm": 0.11492389440536499,
2980
+ "learning_rate": 8.197699835648463e-05,
2981
+ "entropy": 0.006107417472230736,
2982
+ "num_tokens": 18353047.0,
2983
+ "mean_token_accuracy": 0.99817396402359,
2984
+ "epoch": 5.800200050012503,
2985
+ "step": 2900
2986
+ },
2987
+ {
2988
+ "loss": 0.0040702011436223985,
2989
+ "grad_norm": 0.07547246664762497,
2990
+ "learning_rate": 8.132683992789355e-05,
2991
+ "entropy": 0.0047601460755686276,
2992
+ "num_tokens": 18415963.0,
2993
+ "mean_token_accuracy": 0.9987201415002346,
2994
+ "epoch": 5.820205051262816,
2995
+ "step": 2910
2996
+ },
2997
+ {
2998
+ "loss": 0.00443207286298275,
2999
+ "grad_norm": 0.1101204976439476,
3000
+ "learning_rate": 8.067749832381666e-05,
3001
+ "entropy": 0.005160802143291221,
3002
+ "num_tokens": 18479175.0,
3003
+ "mean_token_accuracy": 0.998711721599102,
3004
+ "epoch": 5.840210052513128,
3005
+ "step": 2920
3006
+ },
3007
+ {
3008
+ "loss": 0.004888338595628738,
3009
+ "grad_norm": 0.05996391549706459,
3010
+ "learning_rate": 8.002900194855932e-05,
3011
+ "entropy": 0.006708003048152023,
3012
+ "num_tokens": 18541817.0,
3013
+ "mean_token_accuracy": 0.9980212546885013,
3014
+ "epoch": 5.860215053763441,
3015
+ "step": 2930
3016
+ },
3017
+ {
3018
+ "loss": 0.005104029551148415,
3019
+ "grad_norm": 0.03507474064826965,
3020
+ "learning_rate": 7.938137916945379e-05,
3021
+ "entropy": 0.006489402653096476,
3022
+ "num_tokens": 18604352.0,
3023
+ "mean_token_accuracy": 0.9984742797911167,
3024
+ "epoch": 5.880220055013753,
3025
+ "step": 2940
3026
+ },
3027
+ {
3028
+ "loss": 0.0060115944594144825,
3029
+ "grad_norm": 0.09188883751630783,
3030
+ "learning_rate": 7.873465831561846e-05,
3031
+ "entropy": 0.008611295364971738,
3032
+ "num_tokens": 18667858.0,
3033
+ "mean_token_accuracy": 0.9980850018560886,
3034
+ "epoch": 5.900225056264066,
3035
+ "step": 2950
3036
+ },
3037
+ {
3038
+ "loss": 0.004057048261165619,
3039
+ "grad_norm": 0.1732291430234909,
3040
+ "learning_rate": 7.808886767671858e-05,
3041
+ "entropy": 0.004953707786262385,
3042
+ "num_tokens": 18732898.0,
3043
+ "mean_token_accuracy": 0.9984007500112057,
3044
+ "epoch": 5.920230057514378,
3045
+ "step": 2960
3046
+ },
3047
+ {
3048
+ "loss": 0.006230469048023224,
3049
+ "grad_norm": 0.1265515387058258,
3050
+ "learning_rate": 7.744403550172874e-05,
3051
+ "entropy": 0.005422774016551557,
3052
+ "num_tokens": 18795336.0,
3053
+ "mean_token_accuracy": 0.9979394488036633,
3054
+ "epoch": 5.940235058764691,
3055
+ "step": 2970
3056
+ },
3057
+ {
3058
+ "loss": 0.004861463233828545,
3059
+ "grad_norm": 0.005209980066865683,
3060
+ "learning_rate": 7.680018999769723e-05,
3061
+ "entropy": 0.00592723750633013,
3062
+ "num_tokens": 18856243.0,
3063
+ "mean_token_accuracy": 0.9985147505998612,
3064
+ "epoch": 5.960240060015003,
3065
+ "step": 2980
3066
+ },
3067
+ {
3068
+ "loss": 0.007229304313659668,
3069
+ "grad_norm": 0.30895116925239563,
3070
+ "learning_rate": 7.615735932851223e-05,
3071
+ "entropy": 0.006106495862695738,
3072
+ "num_tokens": 18915043.0,
3073
+ "mean_token_accuracy": 0.9970440402626991,
3074
+ "epoch": 5.980245061265316,
3075
+ "step": 2990
3076
+ },
3077
+ {
3078
+ "loss": 0.004291977360844612,
3079
+ "grad_norm": 0.042590148746967316,
3080
+ "learning_rate": 7.551557161366962e-05,
3081
+ "entropy": 0.0063692813067213645,
3082
+ "num_tokens": 18977028.0,
3083
+ "mean_token_accuracy": 0.9986839317068269,
3084
+ "epoch": 6.0,
3085
+ "step": 3000
3086
+ },
3087
+ {
3088
+ "eval_loss": 0.03871263563632965,
3089
+ "eval_runtime": 68.9977,
3090
+ "eval_samples_per_second": 15.0,
3091
+ "eval_steps_per_second": 7.507,
3092
+ "eval_entropy": 0.014805409190175079,
3093
+ "eval_num_tokens": 18977028.0,
3094
+ "eval_mean_token_accuracy": 0.9913663707644783,
3095
+ "epoch": 6.0,
3096
+ "step": 3000
3097
+ },
3098
+ {
3099
+ "train_runtime": 10605.5798,
3100
+ "train_samples_per_second": 7.541,
3101
+ "train_steps_per_second": 0.471,
3102
+ "total_flos": 4.254846998151168e+16,
3103
+ "train_loss": 0.04043002005169789,
3104
+ "epoch": 6.0,
3105
+ "step": 3000
3106
+ },
3107
+ {
3108
+ "eval_loss": 0.03645886108279228,
3109
+ "eval_runtime": 69.2159,
3110
+ "eval_samples_per_second": 14.953,
3111
+ "eval_steps_per_second": 7.484,
3112
+ "eval_entropy": 0.02839457441272365,
3113
+ "eval_num_tokens": 18977028.0,
3114
+ "eval_mean_token_accuracy": 0.9902446437986661,
3115
+ "epoch": 6.0,
3116
+ "step": 3000
3117
+ }
3118
+ ]
3119
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<|endoftext|>",
5
+ "eos_token": "<|endoftext|>",
6
+ "is_local": true,
7
+ "model_max_length": 2048,
8
+ "pad_token": "<|endoftext|>",
9
+ "tokenizer_class": "TokenizersBackend",
10
+ "unk_token": "<|endoftext|>"
11
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f689d1de06af64c495598bb567c4f842f4f07ab23b514bdea9a9200122abf7d6
3
+ size 5841