add checkpoints/sft3b_lr1e5_bs64

.gitattributes

@@ -125,3 +125,11 @@ checkpoints/sft1.5b_lr1e5_bs32/wandb/offline-run-20260619_141527-f7zbl5g6/run-f7
 checkpoints/sft3b_lr1e5_bs32/checkpoint-3000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoints/sft3b_lr1e5_bs32/wandb/offline-run-20260619_135229-b76ai0c5/run-b76ai0c5.wandb filter=lfs diff=lfs merge=lfs -text
 checkpoints/sft3b_lr1e5_bs32/wandb/offline-run-20260619_141534-lzji1ohf/run-lzji1ohf.wandb filter=lfs diff=lfs merge=lfs -text
+checkpoints/sft3b_lr1e5_bs64/checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/sft3b_lr1e5_bs64/checkpoint-1500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/sft3b_lr1e5_bs64/checkpoint-2000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/sft3b_lr1e5_bs64/checkpoint-2500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/sft3b_lr1e5_bs64/checkpoint-3000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/sft3b_lr1e5_bs64/checkpoint-3468/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/sft3b_lr1e5_bs64/checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/sft3b_lr1e5_bs64/wandb/offline-run-20260620_130707-k94qetn5/run-k94qetn5.wandb filter=lfs diff=lfs merge=lfs -text

checkpoints/sft3b_lr1e5_bs64/checkpoint-1000/added_tokens.json

@@ -0,0 +1,35 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|action_sep|>": 151670,
+  "<|arg_sep|>": 151671,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|call_sep|>": 151666,
+  "<|end_of_text|>": 151673,
+  "<|endoftext|>": 151643,
+  "<|exception_sep|>": 151669,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|frame_sep|>": 151672,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|latent_end|>": 151675,
+  "<|latent_start|>": 151674,
+  "<|line_sep|>": 151667,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|return_sep|>": 151668,
+  "<|trace_context_start|>": 151665,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoints/sft3b_lr1e5_bs64/checkpoint-1000/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

checkpoints/sft3b_lr1e5_bs64/checkpoint-1000/config.json

@@ -0,0 +1,66 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "dtype": "bfloat16",
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 36,
+  "model_type": "qwen2",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 36,
+  "num_key_value_heads": 2,
+  "pad_token_id": 151643,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "transformers_version": "4.57.6",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151676
+}

checkpoints/sft3b_lr1e5_bs64/checkpoint-1000/generation_config.json

@@ -0,0 +1,8 @@
+{
+  "eos_token_id": [
+    151643
+  ],
+  "max_new_tokens": 2048,
+  "pad_token_id": 151643,
+  "transformers_version": "4.57.6"
+}

checkpoints/sft3b_lr1e5_bs64/checkpoint-1000/model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92af380dc43c3e103b97d9071b3b69b34667e6f2a1f2af897ff84248acd06126
+size 4956495344

checkpoints/sft3b_lr1e5_bs64/checkpoint-1000/model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84c085bf2f72f60b2895b144726ac96ba0d9af4a267ced909d3a8fdb8ac8a42c
+size 1214366696

checkpoints/sft3b_lr1e5_bs64/checkpoint-1000/model.safetensors.index.json

@@ -0,0 +1,442 @@
+{
+  "metadata": {
+    "total_parameters": 3085406208,
+    "total_size": 6170812416
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.28.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.28.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

checkpoints/sft3b_lr1e5_bs64/checkpoint-1000/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoints/sft3b_lr1e5_bs64/checkpoint-1000/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83a790d654474f5dfe225f889afd0210313eb1083f942671f2c4b8e95a1c922b
+size 11424004

checkpoints/sft3b_lr1e5_bs64/checkpoint-1000/tokenizer_config.json

@@ -0,0 +1,295 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<|trace_context_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151666": {
+      "content": "<|call_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151667": {
+      "content": "<|line_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151668": {
+      "content": "<|return_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151669": {
+      "content": "<|exception_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151670": {
+      "content": "<|action_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151671": {
+      "content": "<|arg_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151672": {
+      "content": "<|frame_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151673": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151674": {
+      "content": "<|latent_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151675": {
+      "content": "<|latent_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoints/sft3b_lr1e5_bs64/checkpoint-1000/trainer_state.json

@@ -0,0 +1,1434 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.1534135967125658,
+  "eval_steps": 500,
+  "global_step": 1000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.005767428447840819,
+      "grad_norm": 27.875,
+      "learning_rate": 3.80952380952381e-07,
+      "loss": 1.3844,
+      "step": 5
+    },
+    {
+      "epoch": 0.011534856895681638,
+      "grad_norm": 27.75,
+      "learning_rate": 8.571428571428572e-07,
+      "loss": 1.4373,
+      "step": 10
+    },
+    {
+      "epoch": 0.017302285343522458,
+      "grad_norm": 28.625,
+      "learning_rate": 1.3333333333333334e-06,
+      "loss": 1.4472,
+      "step": 15
+    },
+    {
+      "epoch": 0.023069713791363276,
+      "grad_norm": 27.5,
+      "learning_rate": 1.8095238095238097e-06,
+      "loss": 1.4299,
+      "step": 20
+    },
+    {
+      "epoch": 0.028837142239204094,
+      "grad_norm": 21.5,
+      "learning_rate": 2.285714285714286e-06,
+      "loss": 1.3611,
+      "step": 25
+    },
+    {
+      "epoch": 0.034604570687044915,
+      "grad_norm": 14.375,
+      "learning_rate": 2.7619047619047625e-06,
+      "loss": 1.2402,
+      "step": 30
+    },
+    {
+      "epoch": 0.04037199913488573,
+      "grad_norm": 12.8125,
+      "learning_rate": 3.2380952380952385e-06,
+      "loss": 1.1743,
+      "step": 35
+    },
+    {
+      "epoch": 0.04613942758272655,
+      "grad_norm": 11.9375,
+      "learning_rate": 3.7142857142857146e-06,
+      "loss": 1.1619,
+      "step": 40
+    },
+    {
+      "epoch": 0.05190685603056737,
+      "grad_norm": 10.3125,
+      "learning_rate": 4.190476190476191e-06,
+      "loss": 1.0707,
+      "step": 45
+    },
+    {
+      "epoch": 0.05767428447840819,
+      "grad_norm": 9.875,
+      "learning_rate": 4.666666666666667e-06,
+      "loss": 0.9887,
+      "step": 50
+    },
+    {
+      "epoch": 0.063441712926249,
+      "grad_norm": 10.25,
+      "learning_rate": 5.142857142857142e-06,
+      "loss": 0.9292,
+      "step": 55
+    },
+    {
+      "epoch": 0.06920914137408983,
+      "grad_norm": 10.8125,
+      "learning_rate": 5.619047619047619e-06,
+      "loss": 0.8973,
+      "step": 60
+    },
+    {
+      "epoch": 0.07497656982193064,
+      "grad_norm": 9.8125,
+      "learning_rate": 6.095238095238096e-06,
+      "loss": 0.8068,
+      "step": 65
+    },
+    {
+      "epoch": 0.08074399826977147,
+      "grad_norm": 10.75,
+      "learning_rate": 6.571428571428572e-06,
+      "loss": 0.8352,
+      "step": 70
+    },
+    {
+      "epoch": 0.08651142671761229,
+      "grad_norm": 10.0625,
+      "learning_rate": 7.047619047619048e-06,
+      "loss": 0.7974,
+      "step": 75
+    },
+    {
+      "epoch": 0.0922788551654531,
+      "grad_norm": 10.5,
+      "learning_rate": 7.523809523809524e-06,
+      "loss": 0.7557,
+      "step": 80
+    },
+    {
+      "epoch": 0.09804628361329393,
+      "grad_norm": 9.9375,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 0.7018,
+      "step": 85
+    },
+    {
+      "epoch": 0.10381371206113474,
+      "grad_norm": 9.4375,
+      "learning_rate": 8.476190476190477e-06,
+      "loss": 0.688,
+      "step": 90
+    },
+    {
+      "epoch": 0.10958114050897556,
+      "grad_norm": 9.4375,
+      "learning_rate": 8.952380952380953e-06,
+      "loss": 0.6379,
+      "step": 95
+    },
+    {
+      "epoch": 0.11534856895681637,
+      "grad_norm": 8.875,
+      "learning_rate": 9.42857142857143e-06,
+      "loss": 0.5856,
+      "step": 100
+    },
+    {
+      "epoch": 0.1211159974046572,
+      "grad_norm": 9.125,
+      "learning_rate": 9.904761904761906e-06,
+      "loss": 0.5049,
+      "step": 105
+    },
+    {
+      "epoch": 0.126883425852498,
+      "grad_norm": 8.4375,
+      "learning_rate": 9.99996509356498e-06,
+      "loss": 0.4436,
+      "step": 110
+    },
+    {
+      "epoch": 0.13265085430033885,
+      "grad_norm": 7.0,
+      "learning_rate": 9.999823287008022e-06,
+      "loss": 0.3535,
+      "step": 115
+    },
+    {
+      "epoch": 0.13841828274817966,
+      "grad_norm": 5.96875,
+      "learning_rate": 9.999572401768278e-06,
+      "loss": 0.2817,
+      "step": 120
+    },
+    {
+      "epoch": 0.14418571119602047,
+      "grad_norm": 4.625,
+      "learning_rate": 9.999212443319191e-06,
+      "loss": 0.2508,
+      "step": 125
+    },
+    {
+      "epoch": 0.14995313964386128,
+      "grad_norm": 3.5,
+      "learning_rate": 9.998743419513795e-06,
+      "loss": 0.2178,
+      "step": 130
+    },
+    {
+      "epoch": 0.15572056809170212,
+      "grad_norm": 2.21875,
+      "learning_rate": 9.998165340584553e-06,
+      "loss": 0.1906,
+      "step": 135
+    },
+    {
+      "epoch": 0.16148799653954293,
+      "grad_norm": 1.59375,
+      "learning_rate": 9.99747821914313e-06,
+      "loss": 0.1707,
+      "step": 140
+    },
+    {
+      "epoch": 0.16725542498738374,
+      "grad_norm": 1.78125,
+      "learning_rate": 9.99668207018011e-06,
+      "loss": 0.1657,
+      "step": 145
+    },
+    {
+      "epoch": 0.17302285343522458,
+      "grad_norm": 1.71875,
+      "learning_rate": 9.99577691106469e-06,
+      "loss": 0.1475,
+      "step": 150
+    },
+    {
+      "epoch": 0.1787902818830654,
+      "grad_norm": 1.703125,
+      "learning_rate": 9.994762761544276e-06,
+      "loss": 0.139,
+      "step": 155
+    },
+    {
+      "epoch": 0.1845577103309062,
+      "grad_norm": 1.8125,
+      "learning_rate": 9.993639643744071e-06,
+      "loss": 0.1209,
+      "step": 160
+    },
+    {
+      "epoch": 0.19032513877874702,
+      "grad_norm": 1.828125,
+      "learning_rate": 9.992407582166582e-06,
+      "loss": 0.1078,
+      "step": 165
+    },
+    {
+      "epoch": 0.19609256722658785,
+      "grad_norm": 1.8046875,
+      "learning_rate": 9.991066603691092e-06,
+      "loss": 0.0954,
+      "step": 170
+    },
+    {
+      "epoch": 0.20185999567442867,
+      "grad_norm": 1.6484375,
+      "learning_rate": 9.989616737573064e-06,
+      "loss": 0.0848,
+      "step": 175
+    },
+    {
+      "epoch": 0.20762742412226948,
+      "grad_norm": 1.296875,
+      "learning_rate": 9.988058015443517e-06,
+      "loss": 0.0691,
+      "step": 180
+    },
+    {
+      "epoch": 0.2133948525701103,
+      "grad_norm": 1.0234375,
+      "learning_rate": 9.98639047130832e-06,
+      "loss": 0.0602,
+      "step": 185
+    },
+    {
+      "epoch": 0.21916228101795113,
+      "grad_norm": 0.96484375,
+      "learning_rate": 9.984614141547468e-06,
+      "loss": 0.0522,
+      "step": 190
+    },
+    {
+      "epoch": 0.22492970946579194,
+      "grad_norm": 0.77734375,
+      "learning_rate": 9.98272906491427e-06,
+      "loss": 0.0472,
+      "step": 195
+    },
+    {
+      "epoch": 0.23069713791363275,
+      "grad_norm": 0.6328125,
+      "learning_rate": 9.980735282534512e-06,
+      "loss": 0.046,
+      "step": 200
+    },
+    {
+      "epoch": 0.2364645663614736,
+      "grad_norm": 0.7109375,
+      "learning_rate": 9.978632837905566e-06,
+      "loss": 0.0327,
+      "step": 205
+    },
+    {
+      "epoch": 0.2422319948093144,
+      "grad_norm": 0.58984375,
+      "learning_rate": 9.976421776895432e-06,
+      "loss": 0.0341,
+      "step": 210
+    },
+    {
+      "epoch": 0.2479994232571552,
+      "grad_norm": 0.50390625,
+      "learning_rate": 9.974102147741735e-06,
+      "loss": 0.0311,
+      "step": 215
+    },
+    {
+      "epoch": 0.253766851704996,
+      "grad_norm": 0.47265625,
+      "learning_rate": 9.971674001050687e-06,
+      "loss": 0.0284,
+      "step": 220
+    },
+    {
+      "epoch": 0.25953428015283686,
+      "grad_norm": 0.41796875,
+      "learning_rate": 9.969137389795964e-06,
+      "loss": 0.0262,
+      "step": 225
+    },
+    {
+      "epoch": 0.2653017086006777,
+      "grad_norm": 0.5,
+      "learning_rate": 9.966492369317568e-06,
+      "loss": 0.0246,
+      "step": 230
+    },
+    {
+      "epoch": 0.2710691370485185,
+      "grad_norm": 0.40234375,
+      "learning_rate": 9.963738997320609e-06,
+      "loss": 0.0223,
+      "step": 235
+    },
+    {
+      "epoch": 0.2768365654963593,
+      "grad_norm": 0.43359375,
+      "learning_rate": 9.960877333874045e-06,
+      "loss": 0.0237,
+      "step": 240
+    },
+    {
+      "epoch": 0.2826039939442001,
+      "grad_norm": 0.4140625,
+      "learning_rate": 9.95790744140938e-06,
+      "loss": 0.0292,
+      "step": 245
+    },
+    {
+      "epoch": 0.28837142239204094,
+      "grad_norm": 0.39453125,
+      "learning_rate": 9.954829384719296e-06,
+      "loss": 0.0229,
+      "step": 250
+    },
+    {
+      "epoch": 0.2941388508398818,
+      "grad_norm": 0.35546875,
+      "learning_rate": 9.95164323095624e-06,
+      "loss": 0.0198,
+      "step": 255
+    },
+    {
+      "epoch": 0.29990627928772257,
+      "grad_norm": 0.427734375,
+      "learning_rate": 9.948349049630959e-06,
+      "loss": 0.0181,
+      "step": 260
+    },
+    {
+      "epoch": 0.3056737077355634,
+      "grad_norm": 0.35546875,
+      "learning_rate": 9.944946912610986e-06,
+      "loss": 0.0207,
+      "step": 265
+    },
+    {
+      "epoch": 0.31144113618340424,
+      "grad_norm": 0.33984375,
+      "learning_rate": 9.941436894119068e-06,
+      "loss": 0.017,
+      "step": 270
+    },
+    {
+      "epoch": 0.317208564631245,
+      "grad_norm": 0.416015625,
+      "learning_rate": 9.937819070731551e-06,
+      "loss": 0.0157,
+      "step": 275
+    },
+    {
+      "epoch": 0.32297599307908587,
+      "grad_norm": 0.337890625,
+      "learning_rate": 9.934093521376707e-06,
+      "loss": 0.0177,
+      "step": 280
+    },
+    {
+      "epoch": 0.3287434215269267,
+      "grad_norm": 0.328125,
+      "learning_rate": 9.93026032733301e-06,
+      "loss": 0.0164,
+      "step": 285
+    },
+    {
+      "epoch": 0.3345108499747675,
+      "grad_norm": 0.30859375,
+      "learning_rate": 9.926319572227369e-06,
+      "loss": 0.0149,
+      "step": 290
+    },
+    {
+      "epoch": 0.3402782784226083,
+      "grad_norm": 0.31640625,
+      "learning_rate": 9.922271342033295e-06,
+      "loss": 0.0194,
+      "step": 295
+    },
+    {
+      "epoch": 0.34604570687044917,
+      "grad_norm": 0.3515625,
+      "learning_rate": 9.918115725069038e-06,
+      "loss": 0.0152,
+      "step": 300
+    },
+    {
+      "epoch": 0.35181313531828995,
+      "grad_norm": 0.287109375,
+      "learning_rate": 9.913852811995642e-06,
+      "loss": 0.0163,
+      "step": 305
+    },
+    {
+      "epoch": 0.3575805637661308,
+      "grad_norm": 0.37890625,
+      "learning_rate": 9.909482695814986e-06,
+      "loss": 0.0138,
+      "step": 310
+    },
+    {
+      "epoch": 0.36334799221397157,
+      "grad_norm": 0.31640625,
+      "learning_rate": 9.90500547186774e-06,
+      "loss": 0.0174,
+      "step": 315
+    },
+    {
+      "epoch": 0.3691154206618124,
+      "grad_norm": 0.337890625,
+      "learning_rate": 9.900421237831293e-06,
+      "loss": 0.0154,
+      "step": 320
+    },
+    {
+      "epoch": 0.37488284910965325,
+      "grad_norm": 0.2578125,
+      "learning_rate": 9.895730093717629e-06,
+      "loss": 0.0169,
+      "step": 325
+    },
+    {
+      "epoch": 0.38065027755749403,
+      "grad_norm": 0.3359375,
+      "learning_rate": 9.890932141871122e-06,
+      "loss": 0.0151,
+      "step": 330
+    },
+    {
+      "epoch": 0.38641770600533487,
+      "grad_norm": 0.251953125,
+      "learning_rate": 9.886027486966326e-06,
+      "loss": 0.0146,
+      "step": 335
+    },
+    {
+      "epoch": 0.3921851344531757,
+      "grad_norm": 0.314453125,
+      "learning_rate": 9.881016236005686e-06,
+      "loss": 0.0145,
+      "step": 340
+    },
+    {
+      "epoch": 0.3979525629010165,
+      "grad_norm": 0.22265625,
+      "learning_rate": 9.875898498317194e-06,
+      "loss": 0.0136,
+      "step": 345
+    },
+    {
+      "epoch": 0.40371999134885733,
+      "grad_norm": 0.41015625,
+      "learning_rate": 9.870674385552012e-06,
+      "loss": 0.0167,
+      "step": 350
+    },
+    {
+      "epoch": 0.40948741979669817,
+      "grad_norm": 0.271484375,
+      "learning_rate": 9.865344011682038e-06,
+      "loss": 0.0138,
+      "step": 355
+    },
+    {
+      "epoch": 0.41525484824453895,
+      "grad_norm": 0.306640625,
+      "learning_rate": 9.859907492997415e-06,
+      "loss": 0.0135,
+      "step": 360
+    },
+    {
+      "epoch": 0.4210222766923798,
+      "grad_norm": 0.2236328125,
+      "learning_rate": 9.854364948103991e-06,
+      "loss": 0.0127,
+      "step": 365
+    },
+    {
+      "epoch": 0.4267897051402206,
+      "grad_norm": 0.396484375,
+      "learning_rate": 9.848716497920742e-06,
+      "loss": 0.0162,
+      "step": 370
+    },
+    {
+      "epoch": 0.4325571335880614,
+      "grad_norm": 0.33203125,
+      "learning_rate": 9.842962265677123e-06,
+      "loss": 0.0171,
+      "step": 375
+    },
+    {
+      "epoch": 0.43832456203590225,
+      "grad_norm": 0.232421875,
+      "learning_rate": 9.837102376910383e-06,
+      "loss": 0.0131,
+      "step": 380
+    },
+    {
+      "epoch": 0.44409199048374304,
+      "grad_norm": 0.2392578125,
+      "learning_rate": 9.831136959462835e-06,
+      "loss": 0.0134,
+      "step": 385
+    },
+    {
+      "epoch": 0.4498594189315839,
+      "grad_norm": 0.392578125,
+      "learning_rate": 9.825066143479052e-06,
+      "loss": 0.0123,
+      "step": 390
+    },
+    {
+      "epoch": 0.4556268473794247,
+      "grad_norm": 0.53125,
+      "learning_rate": 9.818890061403034e-06,
+      "loss": 0.0169,
+      "step": 395
+    },
+    {
+      "epoch": 0.4613942758272655,
+      "grad_norm": 0.2373046875,
+      "learning_rate": 9.812608847975327e-06,
+      "loss": 0.0136,
+      "step": 400
+    },
+    {
+      "epoch": 0.46716170427510634,
+      "grad_norm": 0.275390625,
+      "learning_rate": 9.80622264023007e-06,
+      "loss": 0.0134,
+      "step": 405
+    },
+    {
+      "epoch": 0.4729291327229472,
+      "grad_norm": 0.240234375,
+      "learning_rate": 9.79973157749201e-06,
+      "loss": 0.0123,
+      "step": 410
+    },
+    {
+      "epoch": 0.47869656117078796,
+      "grad_norm": 0.29296875,
+      "learning_rate": 9.793135801373472e-06,
+      "loss": 0.0125,
+      "step": 415
+    },
+    {
+      "epoch": 0.4844639896186288,
+      "grad_norm": 0.361328125,
+      "learning_rate": 9.786435455771252e-06,
+      "loss": 0.0153,
+      "step": 420
+    },
+    {
+      "epoch": 0.49023141806646964,
+      "grad_norm": 0.5546875,
+      "learning_rate": 9.779630686863493e-06,
+      "loss": 0.0133,
+      "step": 425
+    },
+    {
+      "epoch": 0.4959988465143104,
+      "grad_norm": 0.314453125,
+      "learning_rate": 9.772721643106483e-06,
+      "loss": 0.0142,
+      "step": 430
+    },
+    {
+      "epoch": 0.5017662749621512,
+      "grad_norm": 0.26953125,
+      "learning_rate": 9.765708475231436e-06,
+      "loss": 0.0101,
+      "step": 435
+    },
+    {
+      "epoch": 0.507533703409992,
+      "grad_norm": 0.388671875,
+      "learning_rate": 9.758591336241176e-06,
+      "loss": 0.0255,
+      "step": 440
+    },
+    {
+      "epoch": 0.5133011318578329,
+      "grad_norm": 0.2890625,
+      "learning_rate": 9.75137038140682e-06,
+      "loss": 0.0109,
+      "step": 445
+    },
+    {
+      "epoch": 0.5190685603056737,
+      "grad_norm": 0.1953125,
+      "learning_rate": 9.744045768264387e-06,
+      "loss": 0.0119,
+      "step": 450
+    },
+    {
+      "epoch": 0.5248359887535146,
+      "grad_norm": 0.3125,
+      "learning_rate": 9.736617656611354e-06,
+      "loss": 0.011,
+      "step": 455
+    },
+    {
+      "epoch": 0.5306034172013554,
+      "grad_norm": 0.1982421875,
+      "learning_rate": 9.729086208503174e-06,
+      "loss": 0.0143,
+      "step": 460
+    },
+    {
+      "epoch": 0.5363708456491961,
+      "grad_norm": 0.2275390625,
+      "learning_rate": 9.721451588249744e-06,
+      "loss": 0.0105,
+      "step": 465
+    },
+    {
+      "epoch": 0.542138274097037,
+      "grad_norm": 0.25,
+      "learning_rate": 9.713713962411815e-06,
+      "loss": 0.0165,
+      "step": 470
+    },
+    {
+      "epoch": 0.5479057025448778,
+      "grad_norm": 0.2353515625,
+      "learning_rate": 9.705873499797358e-06,
+      "loss": 0.011,
+      "step": 475
+    },
+    {
+      "epoch": 0.5536731309927186,
+      "grad_norm": 0.23046875,
+      "learning_rate": 9.697930371457886e-06,
+      "loss": 0.011,
+      "step": 480
+    },
+    {
+      "epoch": 0.5594405594405595,
+      "grad_norm": 0.2060546875,
+      "learning_rate": 9.689884750684721e-06,
+      "loss": 0.012,
+      "step": 485
+    },
+    {
+      "epoch": 0.5652079878884002,
+      "grad_norm": 0.349609375,
+      "learning_rate": 9.681736813005207e-06,
+      "loss": 0.0101,
+      "step": 490
+    },
+    {
+      "epoch": 0.570975416336241,
+      "grad_norm": 0.275390625,
+      "learning_rate": 9.673486736178889e-06,
+      "loss": 0.0114,
+      "step": 495
+    },
+    {
+      "epoch": 0.5767428447840819,
+      "grad_norm": 0.193359375,
+      "learning_rate": 9.665134700193633e-06,
+      "loss": 0.0124,
+      "step": 500
+    },
+    {
+      "epoch": 0.5825102732319227,
+      "grad_norm": 0.1904296875,
+      "learning_rate": 9.656680887261693e-06,
+      "loss": 0.0124,
+      "step": 505
+    },
+    {
+      "epoch": 0.5882777016797636,
+      "grad_norm": 0.2265625,
+      "learning_rate": 9.648125481815744e-06,
+      "loss": 0.0151,
+      "step": 510
+    },
+    {
+      "epoch": 0.5940451301276044,
+      "grad_norm": 0.28515625,
+      "learning_rate": 9.639468670504854e-06,
+      "loss": 0.0209,
+      "step": 515
+    },
+    {
+      "epoch": 0.5998125585754451,
+      "grad_norm": 0.171875,
+      "learning_rate": 9.630710642190412e-06,
+      "loss": 0.0107,
+      "step": 520
+    },
+    {
+      "epoch": 0.605579987023286,
+      "grad_norm": 0.1689453125,
+      "learning_rate": 9.621851587942012e-06,
+      "loss": 0.0102,
+      "step": 525
+    },
+    {
+      "epoch": 0.6113474154711268,
+      "grad_norm": 0.224609375,
+      "learning_rate": 9.612891701033278e-06,
+      "loss": 0.013,
+      "step": 530
+    },
+    {
+      "epoch": 0.6171148439189676,
+      "grad_norm": 0.275390625,
+      "learning_rate": 9.603831176937645e-06,
+      "loss": 0.0104,
+      "step": 535
+    },
+    {
+      "epoch": 0.6228822723668085,
+      "grad_norm": 0.197265625,
+      "learning_rate": 9.594670213324114e-06,
+      "loss": 0.0121,
+      "step": 540
+    },
+    {
+      "epoch": 0.6286497008146492,
+      "grad_norm": 0.2021484375,
+      "learning_rate": 9.585409010052913e-06,
+      "loss": 0.012,
+      "step": 545
+    },
+    {
+      "epoch": 0.63441712926249,
+      "grad_norm": 0.255859375,
+      "learning_rate": 9.576047769171154e-06,
+      "loss": 0.0105,
+      "step": 550
+    },
+    {
+      "epoch": 0.6401845577103309,
+      "grad_norm": 0.19140625,
+      "learning_rate": 9.566586694908421e-06,
+      "loss": 0.0106,
+      "step": 555
+    },
+    {
+      "epoch": 0.6459519861581717,
+      "grad_norm": 0.275390625,
+      "learning_rate": 9.557025993672313e-06,
+      "loss": 0.0116,
+      "step": 560
+    },
+    {
+      "epoch": 0.6517194146060126,
+      "grad_norm": 0.2431640625,
+      "learning_rate": 9.547365874043939e-06,
+      "loss": 0.0127,
+      "step": 565
+    },
+    {
+      "epoch": 0.6574868430538534,
+      "grad_norm": 0.1708984375,
+      "learning_rate": 9.537606546773374e-06,
+      "loss": 0.0109,
+      "step": 570
+    },
+    {
+      "epoch": 0.6632542715016941,
+      "grad_norm": 0.30078125,
+      "learning_rate": 9.527748224775054e-06,
+      "loss": 0.0114,
+      "step": 575
+    },
+    {
+      "epoch": 0.669021699949535,
+      "grad_norm": 0.26171875,
+      "learning_rate": 9.517791123123141e-06,
+      "loss": 0.011,
+      "step": 580
+    },
+    {
+      "epoch": 0.6747891283973758,
+      "grad_norm": 0.2041015625,
+      "learning_rate": 9.507735459046812e-06,
+      "loss": 0.0125,
+      "step": 585
+    },
+    {
+      "epoch": 0.6805565568452167,
+      "grad_norm": 0.234375,
+      "learning_rate": 9.497581451925544e-06,
+      "loss": 0.0098,
+      "step": 590
+    },
+    {
+      "epoch": 0.6863239852930575,
+      "grad_norm": 0.2236328125,
+      "learning_rate": 9.487329323284306e-06,
+      "loss": 0.0118,
+      "step": 595
+    },
+    {
+      "epoch": 0.6920914137408983,
+      "grad_norm": 0.2421875,
+      "learning_rate": 9.476979296788746e-06,
+      "loss": 0.0099,
+      "step": 600
+    },
+    {
+      "epoch": 0.6978588421887391,
+      "grad_norm": 0.1982421875,
+      "learning_rate": 9.466531598240292e-06,
+      "loss": 0.014,
+      "step": 605
+    },
+    {
+      "epoch": 0.7036262706365799,
+      "grad_norm": 0.1904296875,
+      "learning_rate": 9.45598645557124e-06,
+      "loss": 0.0104,
+      "step": 610
+    },
+    {
+      "epoch": 0.7093936990844207,
+      "grad_norm": 0.2236328125,
+      "learning_rate": 9.445344098839774e-06,
+      "loss": 0.0106,
+      "step": 615
+    },
+    {
+      "epoch": 0.7151611275322616,
+      "grad_norm": 0.2255859375,
+      "learning_rate": 9.434604760224952e-06,
+      "loss": 0.0116,
+      "step": 620
+    },
+    {
+      "epoch": 0.7209285559801024,
+      "grad_norm": 0.23828125,
+      "learning_rate": 9.423768674021638e-06,
+      "loss": 0.0101,
+      "step": 625
+    },
+    {
+      "epoch": 0.7266959844279431,
+      "grad_norm": 0.1884765625,
+      "learning_rate": 9.412836076635385e-06,
+      "loss": 0.0104,
+      "step": 630
+    },
+    {
+      "epoch": 0.732463412875784,
+      "grad_norm": 0.1982421875,
+      "learning_rate": 9.401807206577292e-06,
+      "loss": 0.0115,
+      "step": 635
+    },
+    {
+      "epoch": 0.7382308413236248,
+      "grad_norm": 0.2265625,
+      "learning_rate": 9.390682304458782e-06,
+      "loss": 0.0098,
+      "step": 640
+    },
+    {
+      "epoch": 0.7439982697714657,
+      "grad_norm": 0.2216796875,
+      "learning_rate": 9.379461612986369e-06,
+      "loss": 0.0118,
+      "step": 645
+    },
+    {
+      "epoch": 0.7497656982193065,
+      "grad_norm": 0.169921875,
+      "learning_rate": 9.368145376956354e-06,
+      "loss": 0.0124,
+      "step": 650
+    },
+    {
+      "epoch": 0.7555331266671473,
+      "grad_norm": 0.1953125,
+      "learning_rate": 9.356733843249487e-06,
+      "loss": 0.0117,
+      "step": 655
+    },
+    {
+      "epoch": 0.7613005551149881,
+      "grad_norm": 0.283203125,
+      "learning_rate": 9.345227260825579e-06,
+      "loss": 0.0104,
+      "step": 660
+    },
+    {
+      "epoch": 0.7670679835628289,
+      "grad_norm": 0.20703125,
+      "learning_rate": 9.333625880718072e-06,
+      "loss": 0.0097,
+      "step": 665
+    },
+    {
+      "epoch": 0.7728354120106697,
+      "grad_norm": 0.1572265625,
+      "learning_rate": 9.321929956028565e-06,
+      "loss": 0.01,
+      "step": 670
+    },
+    {
+      "epoch": 0.7786028404585106,
+      "grad_norm": 0.2080078125,
+      "learning_rate": 9.310139741921287e-06,
+      "loss": 0.0163,
+      "step": 675
+    },
+    {
+      "epoch": 0.7843702689063514,
+      "grad_norm": 0.189453125,
+      "learning_rate": 9.298255495617535e-06,
+      "loss": 0.0102,
+      "step": 680
+    },
+    {
+      "epoch": 0.7901376973541921,
+      "grad_norm": 0.2001953125,
+      "learning_rate": 9.286277476390056e-06,
+      "loss": 0.0091,
+      "step": 685
+    },
+    {
+      "epoch": 0.795905125802033,
+      "grad_norm": 0.1689453125,
+      "learning_rate": 9.274205945557403e-06,
+      "loss": 0.0112,
+      "step": 690
+    },
+    {
+      "epoch": 0.8016725542498738,
+      "grad_norm": 0.330078125,
+      "learning_rate": 9.262041166478215e-06,
+      "loss": 0.0138,
+      "step": 695
+    },
+    {
+      "epoch": 0.8074399826977147,
+      "grad_norm": 0.162109375,
+      "learning_rate": 9.249783404545488e-06,
+      "loss": 0.0083,
+      "step": 700
+    },
+    {
+      "epoch": 0.8132074111455555,
+      "grad_norm": 0.2578125,
+      "learning_rate": 9.23743292718078e-06,
+      "loss": 0.009,
+      "step": 705
+    },
+    {
+      "epoch": 0.8189748395933963,
+      "grad_norm": 0.294921875,
+      "learning_rate": 9.224990003828377e-06,
+      "loss": 0.0119,
+      "step": 710
+    },
+    {
+      "epoch": 0.8247422680412371,
+      "grad_norm": 0.294921875,
+      "learning_rate": 9.212454905949406e-06,
+      "loss": 0.0106,
+      "step": 715
+    },
+    {
+      "epoch": 0.8305096964890779,
+      "grad_norm": 0.1767578125,
+      "learning_rate": 9.199827907015928e-06,
+      "loss": 0.0095,
+      "step": 720
+    },
+    {
+      "epoch": 0.8362771249369187,
+      "grad_norm": 0.1943359375,
+      "learning_rate": 9.187109282504961e-06,
+      "loss": 0.0094,
+      "step": 725
+    },
+    {
+      "epoch": 0.8420445533847596,
+      "grad_norm": 0.2177734375,
+      "learning_rate": 9.174299309892474e-06,
+      "loss": 0.0094,
+      "step": 730
+    },
+    {
+      "epoch": 0.8478119818326004,
+      "grad_norm": 0.2431640625,
+      "learning_rate": 9.161398268647329e-06,
+      "loss": 0.0139,
+      "step": 735
+    },
+    {
+      "epoch": 0.8535794102804412,
+      "grad_norm": 0.2265625,
+      "learning_rate": 9.148406440225193e-06,
+      "loss": 0.0113,
+      "step": 740
+    },
+    {
+      "epoch": 0.859346838728282,
+      "grad_norm": 0.1875,
+      "learning_rate": 9.135324108062391e-06,
+      "loss": 0.0095,
+      "step": 745
+    },
+    {
+      "epoch": 0.8651142671761228,
+      "grad_norm": 0.2041015625,
+      "learning_rate": 9.122151557569719e-06,
+      "loss": 0.009,
+      "step": 750
+    },
+    {
+      "epoch": 0.8708816956239637,
+      "grad_norm": 0.1796875,
+      "learning_rate": 9.108889076126226e-06,
+      "loss": 0.0103,
+      "step": 755
+    },
+    {
+      "epoch": 0.8766491240718045,
+      "grad_norm": 0.28515625,
+      "learning_rate": 9.09553695307294e-06,
+      "loss": 0.0104,
+      "step": 760
+    },
+    {
+      "epoch": 0.8824165525196453,
+      "grad_norm": 0.185546875,
+      "learning_rate": 9.082095479706556e-06,
+      "loss": 0.0084,
+      "step": 765
+    },
+    {
+      "epoch": 0.8881839809674861,
+      "grad_norm": 0.2041015625,
+      "learning_rate": 9.06856494927308e-06,
+      "loss": 0.0096,
+      "step": 770
+    },
+    {
+      "epoch": 0.8939514094153269,
+      "grad_norm": 0.185546875,
+      "learning_rate": 9.054945656961429e-06,
+      "loss": 0.0089,
+      "step": 775
+    },
+    {
+      "epoch": 0.8997188378631678,
+      "grad_norm": 0.15625,
+      "learning_rate": 9.041237899896999e-06,
+      "loss": 0.0087,
+      "step": 780
+    },
+    {
+      "epoch": 0.9054862663110086,
+      "grad_norm": 0.20703125,
+      "learning_rate": 9.027441977135178e-06,
+      "loss": 0.0115,
+      "step": 785
+    },
+    {
+      "epoch": 0.9112536947588494,
+      "grad_norm": 0.271484375,
+      "learning_rate": 9.013558189654819e-06,
+      "loss": 0.009,
+      "step": 790
+    },
+    {
+      "epoch": 0.9170211232066903,
+      "grad_norm": 0.216796875,
+      "learning_rate": 8.999586840351676e-06,
+      "loss": 0.0101,
+      "step": 795
+    },
+    {
+      "epoch": 0.922788551654531,
+      "grad_norm": 0.2041015625,
+      "learning_rate": 8.985528234031799e-06,
+      "loss": 0.0163,
+      "step": 800
+    },
+    {
+      "epoch": 0.9285559801023718,
+      "grad_norm": 0.25,
+      "learning_rate": 8.971382677404878e-06,
+      "loss": 0.0094,
+      "step": 805
+    },
+    {
+      "epoch": 0.9343234085502127,
+      "grad_norm": 0.166015625,
+      "learning_rate": 8.957150479077559e-06,
+      "loss": 0.0074,
+      "step": 810
+    },
+    {
+      "epoch": 0.9400908369980535,
+      "grad_norm": 0.189453125,
+      "learning_rate": 8.942831949546702e-06,
+      "loss": 0.0118,
+      "step": 815
+    },
+    {
+      "epoch": 0.9458582654458944,
+      "grad_norm": 0.2041015625,
+      "learning_rate": 8.928427401192618e-06,
+      "loss": 0.0084,
+      "step": 820
+    },
+    {
+      "epoch": 0.9516256938937351,
+      "grad_norm": 0.203125,
+      "learning_rate": 8.913937148272247e-06,
+      "loss": 0.0082,
+      "step": 825
+    },
+    {
+      "epoch": 0.9573931223415759,
+      "grad_norm": 0.259765625,
+      "learning_rate": 8.899361506912298e-06,
+      "loss": 0.0089,
+      "step": 830
+    },
+    {
+      "epoch": 0.9631605507894168,
+      "grad_norm": 0.1865234375,
+      "learning_rate": 8.884700795102365e-06,
+      "loss": 0.0114,
+      "step": 835
+    },
+    {
+      "epoch": 0.9689279792372576,
+      "grad_norm": 0.208984375,
+      "learning_rate": 8.869955332687975e-06,
+      "loss": 0.0084,
+      "step": 840
+    },
+    {
+      "epoch": 0.9746954076850984,
+      "grad_norm": 0.2197265625,
+      "learning_rate": 8.855125441363628e-06,
+      "loss": 0.0104,
+      "step": 845
+    },
+    {
+      "epoch": 0.9804628361329393,
+      "grad_norm": 0.203125,
+      "learning_rate": 8.840211444665754e-06,
+      "loss": 0.0094,
+      "step": 850
+    },
+    {
+      "epoch": 0.98623026458078,
+      "grad_norm": 0.1884765625,
+      "learning_rate": 8.825213667965678e-06,
+      "loss": 0.0094,
+      "step": 855
+    },
+    {
+      "epoch": 0.9919976930286208,
+      "grad_norm": 0.25390625,
+      "learning_rate": 8.810132438462509e-06,
+      "loss": 0.01,
+      "step": 860
+    },
+    {
+      "epoch": 0.9977651214764617,
+      "grad_norm": 0.263671875,
+      "learning_rate": 8.794968085176006e-06,
+      "loss": 0.0106,
+      "step": 865
+    },
+    {
+      "epoch": 1.0034604570687045,
+      "grad_norm": 0.22265625,
+      "learning_rate": 8.779720938939396e-06,
+      "loss": 0.0124,
+      "step": 870
+    },
+    {
+      "epoch": 1.0092278855165453,
+      "grad_norm": 0.1875,
+      "learning_rate": 8.764391332392163e-06,
+      "loss": 0.0088,
+      "step": 875
+    },
+    {
+      "epoch": 1.0149953139643861,
+      "grad_norm": 0.2294921875,
+      "learning_rate": 8.748979599972787e-06,
+      "loss": 0.0085,
+      "step": 880
+    },
+    {
+      "epoch": 1.020762742412227,
+      "grad_norm": 0.1923828125,
+      "learning_rate": 8.73348607791144e-06,
+      "loss": 0.0086,
+      "step": 885
+    },
+    {
+      "epoch": 1.0265301708600678,
+      "grad_norm": 0.23046875,
+      "learning_rate": 8.717911104222668e-06,
+      "loss": 0.0094,
+      "step": 890
+    },
+    {
+      "epoch": 1.0322975993079087,
+      "grad_norm": 0.1767578125,
+      "learning_rate": 8.702255018698e-06,
+      "loss": 0.0085,
+      "step": 895
+    },
+    {
+      "epoch": 1.0380650277557495,
+      "grad_norm": 0.2080078125,
+      "learning_rate": 8.686518162898547e-06,
+      "loss": 0.0087,
+      "step": 900
+    },
+    {
+      "epoch": 1.0438324562035903,
+      "grad_norm": 0.224609375,
+      "learning_rate": 8.670700880147544e-06,
+      "loss": 0.0088,
+      "step": 905
+    },
+    {
+      "epoch": 1.049599884651431,
+      "grad_norm": 0.2041015625,
+      "learning_rate": 8.65480351552286e-06,
+      "loss": 0.0085,
+      "step": 910
+    },
+    {
+      "epoch": 1.0553673130992718,
+      "grad_norm": 0.1767578125,
+      "learning_rate": 8.638826415849469e-06,
+      "loss": 0.0088,
+      "step": 915
+    },
+    {
+      "epoch": 1.0611347415471126,
+      "grad_norm": 0.158203125,
+      "learning_rate": 8.622769929691889e-06,
+      "loss": 0.011,
+      "step": 920
+    },
+    {
+      "epoch": 1.0669021699949535,
+      "grad_norm": 0.181640625,
+      "learning_rate": 8.606634407346575e-06,
+      "loss": 0.009,
+      "step": 925
+    },
+    {
+      "epoch": 1.0726695984427943,
+      "grad_norm": 0.1806640625,
+      "learning_rate": 8.590420200834271e-06,
+      "loss": 0.0086,
+      "step": 930
+    },
+    {
+      "epoch": 1.0784370268906351,
+      "grad_norm": 0.267578125,
+      "learning_rate": 8.574127663892334e-06,
+      "loss": 0.0093,
+      "step": 935
+    },
+    {
+      "epoch": 1.084204455338476,
+      "grad_norm": 0.228515625,
+      "learning_rate": 8.557757151967025e-06,
+      "loss": 0.0083,
+      "step": 940
+    },
+    {
+      "epoch": 1.0899718837863168,
+      "grad_norm": 0.1904296875,
+      "learning_rate": 8.541309022205738e-06,
+      "loss": 0.0094,
+      "step": 945
+    },
+    {
+      "epoch": 1.0957393122341577,
+      "grad_norm": 0.1806640625,
+      "learning_rate": 8.524783633449224e-06,
+      "loss": 0.0073,
+      "step": 950
+    },
+    {
+      "epoch": 1.1015067406819985,
+      "grad_norm": 0.158203125,
+      "learning_rate": 8.508181346223749e-06,
+      "loss": 0.0076,
+      "step": 955
+    },
+    {
+      "epoch": 1.1072741691298393,
+      "grad_norm": 0.173828125,
+      "learning_rate": 8.491502522733243e-06,
+      "loss": 0.0081,
+      "step": 960
+    },
+    {
+      "epoch": 1.11304159757768,
+      "grad_norm": 0.1845703125,
+      "learning_rate": 8.474747526851383e-06,
+      "loss": 0.007,
+      "step": 965
+    },
+    {
+      "epoch": 1.1188090260255208,
+      "grad_norm": 0.2109375,
+      "learning_rate": 8.457916724113667e-06,
+      "loss": 0.0087,
+      "step": 970
+    },
+    {
+      "epoch": 1.1245764544733616,
+      "grad_norm": 0.1513671875,
+      "learning_rate": 8.441010481709424e-06,
+      "loss": 0.0073,
+      "step": 975
+    },
+    {
+      "epoch": 1.1303438829212025,
+      "grad_norm": 0.1884765625,
+      "learning_rate": 8.424029168473829e-06,
+      "loss": 0.009,
+      "step": 980
+    },
+    {
+      "epoch": 1.1361113113690433,
+      "grad_norm": 0.240234375,
+      "learning_rate": 8.406973154879826e-06,
+      "loss": 0.0085,
+      "step": 985
+    },
+    {
+      "epoch": 1.1418787398168841,
+      "grad_norm": 0.150390625,
+      "learning_rate": 8.389842813030065e-06,
+      "loss": 0.0076,
+      "step": 990
+    },
+    {
+      "epoch": 1.147646168264725,
+      "grad_norm": 0.1923828125,
+      "learning_rate": 8.372638516648782e-06,
+      "loss": 0.0094,
+      "step": 995
+    },
+    {
+      "epoch": 1.1534135967125658,
+      "grad_norm": 0.2099609375,
+      "learning_rate": 8.355360641073637e-06,
+      "loss": 0.0086,
+      "step": 1000
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 3468,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 4,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 8.124685241978716e+17,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoints/sft3b_lr1e5_bs64/checkpoint-1500/added_tokens.json

@@ -0,0 +1,35 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|action_sep|>": 151670,
+  "<|arg_sep|>": 151671,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|call_sep|>": 151666,
+  "<|end_of_text|>": 151673,
+  "<|endoftext|>": 151643,
+  "<|exception_sep|>": 151669,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|frame_sep|>": 151672,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|latent_end|>": 151675,
+  "<|latent_start|>": 151674,
+  "<|line_sep|>": 151667,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|return_sep|>": 151668,
+  "<|trace_context_start|>": 151665,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoints/sft3b_lr1e5_bs64/checkpoint-1500/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

checkpoints/sft3b_lr1e5_bs64/checkpoint-1500/config.json

@@ -0,0 +1,66 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "dtype": "bfloat16",
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 36,
+  "model_type": "qwen2",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 36,
+  "num_key_value_heads": 2,
+  "pad_token_id": 151643,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "transformers_version": "4.57.6",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151676
+}

checkpoints/sft3b_lr1e5_bs64/checkpoint-1500/generation_config.json

@@ -0,0 +1,8 @@
+{
+  "eos_token_id": [
+    151643
+  ],
+  "max_new_tokens": 2048,
+  "pad_token_id": 151643,
+  "transformers_version": "4.57.6"
+}

checkpoints/sft3b_lr1e5_bs64/checkpoint-1500/model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d223a458a3fcb44d6752cf1df245719e17c0f229ff0233bb5d72dfa928abf20d
+size 4956495344

checkpoints/sft3b_lr1e5_bs64/checkpoint-1500/model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b529499d1bf13583814e73e425f2c191e1a02ac92126def83067519e4cb438b
+size 1214366696

checkpoints/sft3b_lr1e5_bs64/checkpoint-1500/model.safetensors.index.json

@@ -0,0 +1,442 @@
+{
+  "metadata": {
+    "total_parameters": 3085406208,
+    "total_size": 6170812416
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.28.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.28.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

checkpoints/sft3b_lr1e5_bs64/checkpoint-1500/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoints/sft3b_lr1e5_bs64/checkpoint-1500/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83a790d654474f5dfe225f889afd0210313eb1083f942671f2c4b8e95a1c922b
+size 11424004

checkpoints/sft3b_lr1e5_bs64/checkpoint-1500/tokenizer_config.json

@@ -0,0 +1,295 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<|trace_context_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151666": {
+      "content": "<|call_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151667": {
+      "content": "<|line_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151668": {
+      "content": "<|return_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151669": {
+      "content": "<|exception_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151670": {
+      "content": "<|action_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151671": {
+      "content": "<|arg_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151672": {
+      "content": "<|frame_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151673": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151674": {
+      "content": "<|latent_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151675": {
+      "content": "<|latent_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoints/sft3b_lr1e5_bs64/checkpoint-1500/trainer_state.json

@@ -0,0 +1,2134 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.7301564414966477,
+  "eval_steps": 500,
+  "global_step": 1500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.005767428447840819,
+      "grad_norm": 27.875,
+      "learning_rate": 3.80952380952381e-07,
+      "loss": 1.3844,
+      "step": 5
+    },
+    {
+      "epoch": 0.011534856895681638,
+      "grad_norm": 27.75,
+      "learning_rate": 8.571428571428572e-07,
+      "loss": 1.4373,
+      "step": 10
+    },
+    {
+      "epoch": 0.017302285343522458,
+      "grad_norm": 28.625,
+      "learning_rate": 1.3333333333333334e-06,
+      "loss": 1.4472,
+      "step": 15
+    },
+    {
+      "epoch": 0.023069713791363276,
+      "grad_norm": 27.5,
+      "learning_rate": 1.8095238095238097e-06,
+      "loss": 1.4299,
+      "step": 20
+    },
+    {
+      "epoch": 0.028837142239204094,
+      "grad_norm": 21.5,
+      "learning_rate": 2.285714285714286e-06,
+      "loss": 1.3611,
+      "step": 25
+    },
+    {
+      "epoch": 0.034604570687044915,
+      "grad_norm": 14.375,
+      "learning_rate": 2.7619047619047625e-06,
+      "loss": 1.2402,
+      "step": 30
+    },
+    {
+      "epoch": 0.04037199913488573,
+      "grad_norm": 12.8125,
+      "learning_rate": 3.2380952380952385e-06,
+      "loss": 1.1743,
+      "step": 35
+    },
+    {
+      "epoch": 0.04613942758272655,
+      "grad_norm": 11.9375,
+      "learning_rate": 3.7142857142857146e-06,
+      "loss": 1.1619,
+      "step": 40
+    },
+    {
+      "epoch": 0.05190685603056737,
+      "grad_norm": 10.3125,
+      "learning_rate": 4.190476190476191e-06,
+      "loss": 1.0707,
+      "step": 45
+    },
+    {
+      "epoch": 0.05767428447840819,
+      "grad_norm": 9.875,
+      "learning_rate": 4.666666666666667e-06,
+      "loss": 0.9887,
+      "step": 50
+    },
+    {
+      "epoch": 0.063441712926249,
+      "grad_norm": 10.25,
+      "learning_rate": 5.142857142857142e-06,
+      "loss": 0.9292,
+      "step": 55
+    },
+    {
+      "epoch": 0.06920914137408983,
+      "grad_norm": 10.8125,
+      "learning_rate": 5.619047619047619e-06,
+      "loss": 0.8973,
+      "step": 60
+    },
+    {
+      "epoch": 0.07497656982193064,
+      "grad_norm": 9.8125,
+      "learning_rate": 6.095238095238096e-06,
+      "loss": 0.8068,
+      "step": 65
+    },
+    {
+      "epoch": 0.08074399826977147,
+      "grad_norm": 10.75,
+      "learning_rate": 6.571428571428572e-06,
+      "loss": 0.8352,
+      "step": 70
+    },
+    {
+      "epoch": 0.08651142671761229,
+      "grad_norm": 10.0625,
+      "learning_rate": 7.047619047619048e-06,
+      "loss": 0.7974,
+      "step": 75
+    },
+    {
+      "epoch": 0.0922788551654531,
+      "grad_norm": 10.5,
+      "learning_rate": 7.523809523809524e-06,
+      "loss": 0.7557,
+      "step": 80
+    },
+    {
+      "epoch": 0.09804628361329393,
+      "grad_norm": 9.9375,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 0.7018,
+      "step": 85
+    },
+    {
+      "epoch": 0.10381371206113474,
+      "grad_norm": 9.4375,
+      "learning_rate": 8.476190476190477e-06,
+      "loss": 0.688,
+      "step": 90
+    },
+    {
+      "epoch": 0.10958114050897556,
+      "grad_norm": 9.4375,
+      "learning_rate": 8.952380952380953e-06,
+      "loss": 0.6379,
+      "step": 95
+    },
+    {
+      "epoch": 0.11534856895681637,
+      "grad_norm": 8.875,
+      "learning_rate": 9.42857142857143e-06,
+      "loss": 0.5856,
+      "step": 100
+    },
+    {
+      "epoch": 0.1211159974046572,
+      "grad_norm": 9.125,
+      "learning_rate": 9.904761904761906e-06,
+      "loss": 0.5049,
+      "step": 105
+    },
+    {
+      "epoch": 0.126883425852498,
+      "grad_norm": 8.4375,
+      "learning_rate": 9.99996509356498e-06,
+      "loss": 0.4436,
+      "step": 110
+    },
+    {
+      "epoch": 0.13265085430033885,
+      "grad_norm": 7.0,
+      "learning_rate": 9.999823287008022e-06,
+      "loss": 0.3535,
+      "step": 115
+    },
+    {
+      "epoch": 0.13841828274817966,
+      "grad_norm": 5.96875,
+      "learning_rate": 9.999572401768278e-06,
+      "loss": 0.2817,
+      "step": 120
+    },
+    {
+      "epoch": 0.14418571119602047,
+      "grad_norm": 4.625,
+      "learning_rate": 9.999212443319191e-06,
+      "loss": 0.2508,
+      "step": 125
+    },
+    {
+      "epoch": 0.14995313964386128,
+      "grad_norm": 3.5,
+      "learning_rate": 9.998743419513795e-06,
+      "loss": 0.2178,
+      "step": 130
+    },
+    {
+      "epoch": 0.15572056809170212,
+      "grad_norm": 2.21875,
+      "learning_rate": 9.998165340584553e-06,
+      "loss": 0.1906,
+      "step": 135
+    },
+    {
+      "epoch": 0.16148799653954293,
+      "grad_norm": 1.59375,
+      "learning_rate": 9.99747821914313e-06,
+      "loss": 0.1707,
+      "step": 140
+    },
+    {
+      "epoch": 0.16725542498738374,
+      "grad_norm": 1.78125,
+      "learning_rate": 9.99668207018011e-06,
+      "loss": 0.1657,
+      "step": 145
+    },
+    {
+      "epoch": 0.17302285343522458,
+      "grad_norm": 1.71875,
+      "learning_rate": 9.99577691106469e-06,
+      "loss": 0.1475,
+      "step": 150
+    },
+    {
+      "epoch": 0.1787902818830654,
+      "grad_norm": 1.703125,
+      "learning_rate": 9.994762761544276e-06,
+      "loss": 0.139,
+      "step": 155
+    },
+    {
+      "epoch": 0.1845577103309062,
+      "grad_norm": 1.8125,
+      "learning_rate": 9.993639643744071e-06,
+      "loss": 0.1209,
+      "step": 160
+    },
+    {
+      "epoch": 0.19032513877874702,
+      "grad_norm": 1.828125,
+      "learning_rate": 9.992407582166582e-06,
+      "loss": 0.1078,
+      "step": 165
+    },
+    {
+      "epoch": 0.19609256722658785,
+      "grad_norm": 1.8046875,
+      "learning_rate": 9.991066603691092e-06,
+      "loss": 0.0954,
+      "step": 170
+    },
+    {
+      "epoch": 0.20185999567442867,
+      "grad_norm": 1.6484375,
+      "learning_rate": 9.989616737573064e-06,
+      "loss": 0.0848,
+      "step": 175
+    },
+    {
+      "epoch": 0.20762742412226948,
+      "grad_norm": 1.296875,
+      "learning_rate": 9.988058015443517e-06,
+      "loss": 0.0691,
+      "step": 180
+    },
+    {
+      "epoch": 0.2133948525701103,
+      "grad_norm": 1.0234375,
+      "learning_rate": 9.98639047130832e-06,
+      "loss": 0.0602,
+      "step": 185
+    },
+    {
+      "epoch": 0.21916228101795113,
+      "grad_norm": 0.96484375,
+      "learning_rate": 9.984614141547468e-06,
+      "loss": 0.0522,
+      "step": 190
+    },
+    {
+      "epoch": 0.22492970946579194,
+      "grad_norm": 0.77734375,
+      "learning_rate": 9.98272906491427e-06,
+      "loss": 0.0472,
+      "step": 195
+    },
+    {
+      "epoch": 0.23069713791363275,
+      "grad_norm": 0.6328125,
+      "learning_rate": 9.980735282534512e-06,
+      "loss": 0.046,
+      "step": 200
+    },
+    {
+      "epoch": 0.2364645663614736,
+      "grad_norm": 0.7109375,
+      "learning_rate": 9.978632837905566e-06,
+      "loss": 0.0327,
+      "step": 205
+    },
+    {
+      "epoch": 0.2422319948093144,
+      "grad_norm": 0.58984375,
+      "learning_rate": 9.976421776895432e-06,
+      "loss": 0.0341,
+      "step": 210
+    },
+    {
+      "epoch": 0.2479994232571552,
+      "grad_norm": 0.50390625,
+      "learning_rate": 9.974102147741735e-06,
+      "loss": 0.0311,
+      "step": 215
+    },
+    {
+      "epoch": 0.253766851704996,
+      "grad_norm": 0.47265625,
+      "learning_rate": 9.971674001050687e-06,
+      "loss": 0.0284,
+      "step": 220
+    },
+    {
+      "epoch": 0.25953428015283686,
+      "grad_norm": 0.41796875,
+      "learning_rate": 9.969137389795964e-06,
+      "loss": 0.0262,
+      "step": 225
+    },
+    {
+      "epoch": 0.2653017086006777,
+      "grad_norm": 0.5,
+      "learning_rate": 9.966492369317568e-06,
+      "loss": 0.0246,
+      "step": 230
+    },
+    {
+      "epoch": 0.2710691370485185,
+      "grad_norm": 0.40234375,
+      "learning_rate": 9.963738997320609e-06,
+      "loss": 0.0223,
+      "step": 235
+    },
+    {
+      "epoch": 0.2768365654963593,
+      "grad_norm": 0.43359375,
+      "learning_rate": 9.960877333874045e-06,
+      "loss": 0.0237,
+      "step": 240
+    },
+    {
+      "epoch": 0.2826039939442001,
+      "grad_norm": 0.4140625,
+      "learning_rate": 9.95790744140938e-06,
+      "loss": 0.0292,
+      "step": 245
+    },
+    {
+      "epoch": 0.28837142239204094,
+      "grad_norm": 0.39453125,
+      "learning_rate": 9.954829384719296e-06,
+      "loss": 0.0229,
+      "step": 250
+    },
+    {
+      "epoch": 0.2941388508398818,
+      "grad_norm": 0.35546875,
+      "learning_rate": 9.95164323095624e-06,
+      "loss": 0.0198,
+      "step": 255
+    },
+    {
+      "epoch": 0.29990627928772257,
+      "grad_norm": 0.427734375,
+      "learning_rate": 9.948349049630959e-06,
+      "loss": 0.0181,
+      "step": 260
+    },
+    {
+      "epoch": 0.3056737077355634,
+      "grad_norm": 0.35546875,
+      "learning_rate": 9.944946912610986e-06,
+      "loss": 0.0207,
+      "step": 265
+    },
+    {
+      "epoch": 0.31144113618340424,
+      "grad_norm": 0.33984375,
+      "learning_rate": 9.941436894119068e-06,
+      "loss": 0.017,
+      "step": 270
+    },
+    {
+      "epoch": 0.317208564631245,
+      "grad_norm": 0.416015625,
+      "learning_rate": 9.937819070731551e-06,
+      "loss": 0.0157,
+      "step": 275
+    },
+    {
+      "epoch": 0.32297599307908587,
+      "grad_norm": 0.337890625,
+      "learning_rate": 9.934093521376707e-06,
+      "loss": 0.0177,
+      "step": 280
+    },
+    {
+      "epoch": 0.3287434215269267,
+      "grad_norm": 0.328125,
+      "learning_rate": 9.93026032733301e-06,
+      "loss": 0.0164,
+      "step": 285
+    },
+    {
+      "epoch": 0.3345108499747675,
+      "grad_norm": 0.30859375,
+      "learning_rate": 9.926319572227369e-06,
+      "loss": 0.0149,
+      "step": 290
+    },
+    {
+      "epoch": 0.3402782784226083,
+      "grad_norm": 0.31640625,
+      "learning_rate": 9.922271342033295e-06,
+      "loss": 0.0194,
+      "step": 295
+    },
+    {
+      "epoch": 0.34604570687044917,
+      "grad_norm": 0.3515625,
+      "learning_rate": 9.918115725069038e-06,
+      "loss": 0.0152,
+      "step": 300
+    },
+    {
+      "epoch": 0.35181313531828995,
+      "grad_norm": 0.287109375,
+      "learning_rate": 9.913852811995642e-06,
+      "loss": 0.0163,
+      "step": 305
+    },
+    {
+      "epoch": 0.3575805637661308,
+      "grad_norm": 0.37890625,
+      "learning_rate": 9.909482695814986e-06,
+      "loss": 0.0138,
+      "step": 310
+    },
+    {
+      "epoch": 0.36334799221397157,
+      "grad_norm": 0.31640625,
+      "learning_rate": 9.90500547186774e-06,
+      "loss": 0.0174,
+      "step": 315
+    },
+    {
+      "epoch": 0.3691154206618124,
+      "grad_norm": 0.337890625,
+      "learning_rate": 9.900421237831293e-06,
+      "loss": 0.0154,
+      "step": 320
+    },
+    {
+      "epoch": 0.37488284910965325,
+      "grad_norm": 0.2578125,
+      "learning_rate": 9.895730093717629e-06,
+      "loss": 0.0169,
+      "step": 325
+    },
+    {
+      "epoch": 0.38065027755749403,
+      "grad_norm": 0.3359375,
+      "learning_rate": 9.890932141871122e-06,
+      "loss": 0.0151,
+      "step": 330
+    },
+    {
+      "epoch": 0.38641770600533487,
+      "grad_norm": 0.251953125,
+      "learning_rate": 9.886027486966326e-06,
+      "loss": 0.0146,
+      "step": 335
+    },
+    {
+      "epoch": 0.3921851344531757,
+      "grad_norm": 0.314453125,
+      "learning_rate": 9.881016236005686e-06,
+      "loss": 0.0145,
+      "step": 340
+    },
+    {
+      "epoch": 0.3979525629010165,
+      "grad_norm": 0.22265625,
+      "learning_rate": 9.875898498317194e-06,
+      "loss": 0.0136,
+      "step": 345
+    },
+    {
+      "epoch": 0.40371999134885733,
+      "grad_norm": 0.41015625,
+      "learning_rate": 9.870674385552012e-06,
+      "loss": 0.0167,
+      "step": 350
+    },
+    {
+      "epoch": 0.40948741979669817,
+      "grad_norm": 0.271484375,
+      "learning_rate": 9.865344011682038e-06,
+      "loss": 0.0138,
+      "step": 355
+    },
+    {
+      "epoch": 0.41525484824453895,
+      "grad_norm": 0.306640625,
+      "learning_rate": 9.859907492997415e-06,
+      "loss": 0.0135,
+      "step": 360
+    },
+    {
+      "epoch": 0.4210222766923798,
+      "grad_norm": 0.2236328125,
+      "learning_rate": 9.854364948103991e-06,
+      "loss": 0.0127,
+      "step": 365
+    },
+    {
+      "epoch": 0.4267897051402206,
+      "grad_norm": 0.396484375,
+      "learning_rate": 9.848716497920742e-06,
+      "loss": 0.0162,
+      "step": 370
+    },
+    {
+      "epoch": 0.4325571335880614,
+      "grad_norm": 0.33203125,
+      "learning_rate": 9.842962265677123e-06,
+      "loss": 0.0171,
+      "step": 375
+    },
+    {
+      "epoch": 0.43832456203590225,
+      "grad_norm": 0.232421875,
+      "learning_rate": 9.837102376910383e-06,
+      "loss": 0.0131,
+      "step": 380
+    },
+    {
+      "epoch": 0.44409199048374304,
+      "grad_norm": 0.2392578125,
+      "learning_rate": 9.831136959462835e-06,
+      "loss": 0.0134,
+      "step": 385
+    },
+    {
+      "epoch": 0.4498594189315839,
+      "grad_norm": 0.392578125,
+      "learning_rate": 9.825066143479052e-06,
+      "loss": 0.0123,
+      "step": 390
+    },
+    {
+      "epoch": 0.4556268473794247,
+      "grad_norm": 0.53125,
+      "learning_rate": 9.818890061403034e-06,
+      "loss": 0.0169,
+      "step": 395
+    },
+    {
+      "epoch": 0.4613942758272655,
+      "grad_norm": 0.2373046875,
+      "learning_rate": 9.812608847975327e-06,
+      "loss": 0.0136,
+      "step": 400
+    },
+    {
+      "epoch": 0.46716170427510634,
+      "grad_norm": 0.275390625,
+      "learning_rate": 9.80622264023007e-06,
+      "loss": 0.0134,
+      "step": 405
+    },
+    {
+      "epoch": 0.4729291327229472,
+      "grad_norm": 0.240234375,
+      "learning_rate": 9.79973157749201e-06,
+      "loss": 0.0123,
+      "step": 410
+    },
+    {
+      "epoch": 0.47869656117078796,
+      "grad_norm": 0.29296875,
+      "learning_rate": 9.793135801373472e-06,
+      "loss": 0.0125,
+      "step": 415
+    },
+    {
+      "epoch": 0.4844639896186288,
+      "grad_norm": 0.361328125,
+      "learning_rate": 9.786435455771252e-06,
+      "loss": 0.0153,
+      "step": 420
+    },
+    {
+      "epoch": 0.49023141806646964,
+      "grad_norm": 0.5546875,
+      "learning_rate": 9.779630686863493e-06,
+      "loss": 0.0133,
+      "step": 425
+    },
+    {
+      "epoch": 0.4959988465143104,
+      "grad_norm": 0.314453125,
+      "learning_rate": 9.772721643106483e-06,
+      "loss": 0.0142,
+      "step": 430
+    },
+    {
+      "epoch": 0.5017662749621512,
+      "grad_norm": 0.26953125,
+      "learning_rate": 9.765708475231436e-06,
+      "loss": 0.0101,
+      "step": 435
+    },
+    {
+      "epoch": 0.507533703409992,
+      "grad_norm": 0.388671875,
+      "learning_rate": 9.758591336241176e-06,
+      "loss": 0.0255,
+      "step": 440
+    },
+    {
+      "epoch": 0.5133011318578329,
+      "grad_norm": 0.2890625,
+      "learning_rate": 9.75137038140682e-06,
+      "loss": 0.0109,
+      "step": 445
+    },
+    {
+      "epoch": 0.5190685603056737,
+      "grad_norm": 0.1953125,
+      "learning_rate": 9.744045768264387e-06,
+      "loss": 0.0119,
+      "step": 450
+    },
+    {
+      "epoch": 0.5248359887535146,
+      "grad_norm": 0.3125,
+      "learning_rate": 9.736617656611354e-06,
+      "loss": 0.011,
+      "step": 455
+    },
+    {
+      "epoch": 0.5306034172013554,
+      "grad_norm": 0.1982421875,
+      "learning_rate": 9.729086208503174e-06,
+      "loss": 0.0143,
+      "step": 460
+    },
+    {
+      "epoch": 0.5363708456491961,
+      "grad_norm": 0.2275390625,
+      "learning_rate": 9.721451588249744e-06,
+      "loss": 0.0105,
+      "step": 465
+    },
+    {
+      "epoch": 0.542138274097037,
+      "grad_norm": 0.25,
+      "learning_rate": 9.713713962411815e-06,
+      "loss": 0.0165,
+      "step": 470
+    },
+    {
+      "epoch": 0.5479057025448778,
+      "grad_norm": 0.2353515625,
+      "learning_rate": 9.705873499797358e-06,
+      "loss": 0.011,
+      "step": 475
+    },
+    {
+      "epoch": 0.5536731309927186,
+      "grad_norm": 0.23046875,
+      "learning_rate": 9.697930371457886e-06,
+      "loss": 0.011,
+      "step": 480
+    },
+    {
+      "epoch": 0.5594405594405595,
+      "grad_norm": 0.2060546875,
+      "learning_rate": 9.689884750684721e-06,
+      "loss": 0.012,
+      "step": 485
+    },
+    {
+      "epoch": 0.5652079878884002,
+      "grad_norm": 0.349609375,
+      "learning_rate": 9.681736813005207e-06,
+      "loss": 0.0101,
+      "step": 490
+    },
+    {
+      "epoch": 0.570975416336241,
+      "grad_norm": 0.275390625,
+      "learning_rate": 9.673486736178889e-06,
+      "loss": 0.0114,
+      "step": 495
+    },
+    {
+      "epoch": 0.5767428447840819,
+      "grad_norm": 0.193359375,
+      "learning_rate": 9.665134700193633e-06,
+      "loss": 0.0124,
+      "step": 500
+    },
+    {
+      "epoch": 0.5825102732319227,
+      "grad_norm": 0.1904296875,
+      "learning_rate": 9.656680887261693e-06,
+      "loss": 0.0124,
+      "step": 505
+    },
+    {
+      "epoch": 0.5882777016797636,
+      "grad_norm": 0.2265625,
+      "learning_rate": 9.648125481815744e-06,
+      "loss": 0.0151,
+      "step": 510
+    },
+    {
+      "epoch": 0.5940451301276044,
+      "grad_norm": 0.28515625,
+      "learning_rate": 9.639468670504854e-06,
+      "loss": 0.0209,
+      "step": 515
+    },
+    {
+      "epoch": 0.5998125585754451,
+      "grad_norm": 0.171875,
+      "learning_rate": 9.630710642190412e-06,
+      "loss": 0.0107,
+      "step": 520
+    },
+    {
+      "epoch": 0.605579987023286,
+      "grad_norm": 0.1689453125,
+      "learning_rate": 9.621851587942012e-06,
+      "loss": 0.0102,
+      "step": 525
+    },
+    {
+      "epoch": 0.6113474154711268,
+      "grad_norm": 0.224609375,
+      "learning_rate": 9.612891701033278e-06,
+      "loss": 0.013,
+      "step": 530
+    },
+    {
+      "epoch": 0.6171148439189676,
+      "grad_norm": 0.275390625,
+      "learning_rate": 9.603831176937645e-06,
+      "loss": 0.0104,
+      "step": 535
+    },
+    {
+      "epoch": 0.6228822723668085,
+      "grad_norm": 0.197265625,
+      "learning_rate": 9.594670213324114e-06,
+      "loss": 0.0121,
+      "step": 540
+    },
+    {
+      "epoch": 0.6286497008146492,
+      "grad_norm": 0.2021484375,
+      "learning_rate": 9.585409010052913e-06,
+      "loss": 0.012,
+      "step": 545
+    },
+    {
+      "epoch": 0.63441712926249,
+      "grad_norm": 0.255859375,
+      "learning_rate": 9.576047769171154e-06,
+      "loss": 0.0105,
+      "step": 550
+    },
+    {
+      "epoch": 0.6401845577103309,
+      "grad_norm": 0.19140625,
+      "learning_rate": 9.566586694908421e-06,
+      "loss": 0.0106,
+      "step": 555
+    },
+    {
+      "epoch": 0.6459519861581717,
+      "grad_norm": 0.275390625,
+      "learning_rate": 9.557025993672313e-06,
+      "loss": 0.0116,
+      "step": 560
+    },
+    {
+      "epoch": 0.6517194146060126,
+      "grad_norm": 0.2431640625,
+      "learning_rate": 9.547365874043939e-06,
+      "loss": 0.0127,
+      "step": 565
+    },
+    {
+      "epoch": 0.6574868430538534,
+      "grad_norm": 0.1708984375,
+      "learning_rate": 9.537606546773374e-06,
+      "loss": 0.0109,
+      "step": 570
+    },
+    {
+      "epoch": 0.6632542715016941,
+      "grad_norm": 0.30078125,
+      "learning_rate": 9.527748224775054e-06,
+      "loss": 0.0114,
+      "step": 575
+    },
+    {
+      "epoch": 0.669021699949535,
+      "grad_norm": 0.26171875,
+      "learning_rate": 9.517791123123141e-06,
+      "loss": 0.011,
+      "step": 580
+    },
+    {
+      "epoch": 0.6747891283973758,
+      "grad_norm": 0.2041015625,
+      "learning_rate": 9.507735459046812e-06,
+      "loss": 0.0125,
+      "step": 585
+    },
+    {
+      "epoch": 0.6805565568452167,
+      "grad_norm": 0.234375,
+      "learning_rate": 9.497581451925544e-06,
+      "loss": 0.0098,
+      "step": 590
+    },
+    {
+      "epoch": 0.6863239852930575,
+      "grad_norm": 0.2236328125,
+      "learning_rate": 9.487329323284306e-06,
+      "loss": 0.0118,
+      "step": 595
+    },
+    {
+      "epoch": 0.6920914137408983,
+      "grad_norm": 0.2421875,
+      "learning_rate": 9.476979296788746e-06,
+      "loss": 0.0099,
+      "step": 600
+    },
+    {
+      "epoch": 0.6978588421887391,
+      "grad_norm": 0.1982421875,
+      "learning_rate": 9.466531598240292e-06,
+      "loss": 0.014,
+      "step": 605
+    },
+    {
+      "epoch": 0.7036262706365799,
+      "grad_norm": 0.1904296875,
+      "learning_rate": 9.45598645557124e-06,
+      "loss": 0.0104,
+      "step": 610
+    },
+    {
+      "epoch": 0.7093936990844207,
+      "grad_norm": 0.2236328125,
+      "learning_rate": 9.445344098839774e-06,
+      "loss": 0.0106,
+      "step": 615
+    },
+    {
+      "epoch": 0.7151611275322616,
+      "grad_norm": 0.2255859375,
+      "learning_rate": 9.434604760224952e-06,
+      "loss": 0.0116,
+      "step": 620
+    },
+    {
+      "epoch": 0.7209285559801024,
+      "grad_norm": 0.23828125,
+      "learning_rate": 9.423768674021638e-06,
+      "loss": 0.0101,
+      "step": 625
+    },
+    {
+      "epoch": 0.7266959844279431,
+      "grad_norm": 0.1884765625,
+      "learning_rate": 9.412836076635385e-06,
+      "loss": 0.0104,
+      "step": 630
+    },
+    {
+      "epoch": 0.732463412875784,
+      "grad_norm": 0.1982421875,
+      "learning_rate": 9.401807206577292e-06,
+      "loss": 0.0115,
+      "step": 635
+    },
+    {
+      "epoch": 0.7382308413236248,
+      "grad_norm": 0.2265625,
+      "learning_rate": 9.390682304458782e-06,
+      "loss": 0.0098,
+      "step": 640
+    },
+    {
+      "epoch": 0.7439982697714657,
+      "grad_norm": 0.2216796875,
+      "learning_rate": 9.379461612986369e-06,
+      "loss": 0.0118,
+      "step": 645
+    },
+    {
+      "epoch": 0.7497656982193065,
+      "grad_norm": 0.169921875,
+      "learning_rate": 9.368145376956354e-06,
+      "loss": 0.0124,
+      "step": 650
+    },
+    {
+      "epoch": 0.7555331266671473,
+      "grad_norm": 0.1953125,
+      "learning_rate": 9.356733843249487e-06,
+      "loss": 0.0117,
+      "step": 655
+    },
+    {
+      "epoch": 0.7613005551149881,
+      "grad_norm": 0.283203125,
+      "learning_rate": 9.345227260825579e-06,
+      "loss": 0.0104,
+      "step": 660
+    },
+    {
+      "epoch": 0.7670679835628289,
+      "grad_norm": 0.20703125,
+      "learning_rate": 9.333625880718072e-06,
+      "loss": 0.0097,
+      "step": 665
+    },
+    {
+      "epoch": 0.7728354120106697,
+      "grad_norm": 0.1572265625,
+      "learning_rate": 9.321929956028565e-06,
+      "loss": 0.01,
+      "step": 670
+    },
+    {
+      "epoch": 0.7786028404585106,
+      "grad_norm": 0.2080078125,
+      "learning_rate": 9.310139741921287e-06,
+      "loss": 0.0163,
+      "step": 675
+    },
+    {
+      "epoch": 0.7843702689063514,
+      "grad_norm": 0.189453125,
+      "learning_rate": 9.298255495617535e-06,
+      "loss": 0.0102,
+      "step": 680
+    },
+    {
+      "epoch": 0.7901376973541921,
+      "grad_norm": 0.2001953125,
+      "learning_rate": 9.286277476390056e-06,
+      "loss": 0.0091,
+      "step": 685
+    },
+    {
+      "epoch": 0.795905125802033,
+      "grad_norm": 0.1689453125,
+      "learning_rate": 9.274205945557403e-06,
+      "loss": 0.0112,
+      "step": 690
+    },
+    {
+      "epoch": 0.8016725542498738,
+      "grad_norm": 0.330078125,
+      "learning_rate": 9.262041166478215e-06,
+      "loss": 0.0138,
+      "step": 695
+    },
+    {
+      "epoch": 0.8074399826977147,
+      "grad_norm": 0.162109375,
+      "learning_rate": 9.249783404545488e-06,
+      "loss": 0.0083,
+      "step": 700
+    },
+    {
+      "epoch": 0.8132074111455555,
+      "grad_norm": 0.2578125,
+      "learning_rate": 9.23743292718078e-06,
+      "loss": 0.009,
+      "step": 705
+    },
+    {
+      "epoch": 0.8189748395933963,
+      "grad_norm": 0.294921875,
+      "learning_rate": 9.224990003828377e-06,
+      "loss": 0.0119,
+      "step": 710
+    },
+    {
+      "epoch": 0.8247422680412371,
+      "grad_norm": 0.294921875,
+      "learning_rate": 9.212454905949406e-06,
+      "loss": 0.0106,
+      "step": 715
+    },
+    {
+      "epoch": 0.8305096964890779,
+      "grad_norm": 0.1767578125,
+      "learning_rate": 9.199827907015928e-06,
+      "loss": 0.0095,
+      "step": 720
+    },
+    {
+      "epoch": 0.8362771249369187,
+      "grad_norm": 0.1943359375,
+      "learning_rate": 9.187109282504961e-06,
+      "loss": 0.0094,
+      "step": 725
+    },
+    {
+      "epoch": 0.8420445533847596,
+      "grad_norm": 0.2177734375,
+      "learning_rate": 9.174299309892474e-06,
+      "loss": 0.0094,
+      "step": 730
+    },
+    {
+      "epoch": 0.8478119818326004,
+      "grad_norm": 0.2431640625,
+      "learning_rate": 9.161398268647329e-06,
+      "loss": 0.0139,
+      "step": 735
+    },
+    {
+      "epoch": 0.8535794102804412,
+      "grad_norm": 0.2265625,
+      "learning_rate": 9.148406440225193e-06,
+      "loss": 0.0113,
+      "step": 740
+    },
+    {
+      "epoch": 0.859346838728282,
+      "grad_norm": 0.1875,
+      "learning_rate": 9.135324108062391e-06,
+      "loss": 0.0095,
+      "step": 745
+    },
+    {
+      "epoch": 0.8651142671761228,
+      "grad_norm": 0.2041015625,
+      "learning_rate": 9.122151557569719e-06,
+      "loss": 0.009,
+      "step": 750
+    },
+    {
+      "epoch": 0.8708816956239637,
+      "grad_norm": 0.1796875,
+      "learning_rate": 9.108889076126226e-06,
+      "loss": 0.0103,
+      "step": 755
+    },
+    {
+      "epoch": 0.8766491240718045,
+      "grad_norm": 0.28515625,
+      "learning_rate": 9.09553695307294e-06,
+      "loss": 0.0104,
+      "step": 760
+    },
+    {
+      "epoch": 0.8824165525196453,
+      "grad_norm": 0.185546875,
+      "learning_rate": 9.082095479706556e-06,
+      "loss": 0.0084,
+      "step": 765
+    },
+    {
+      "epoch": 0.8881839809674861,
+      "grad_norm": 0.2041015625,
+      "learning_rate": 9.06856494927308e-06,
+      "loss": 0.0096,
+      "step": 770
+    },
+    {
+      "epoch": 0.8939514094153269,
+      "grad_norm": 0.185546875,
+      "learning_rate": 9.054945656961429e-06,
+      "loss": 0.0089,
+      "step": 775
+    },
+    {
+      "epoch": 0.8997188378631678,
+      "grad_norm": 0.15625,
+      "learning_rate": 9.041237899896999e-06,
+      "loss": 0.0087,
+      "step": 780
+    },
+    {
+      "epoch": 0.9054862663110086,
+      "grad_norm": 0.20703125,
+      "learning_rate": 9.027441977135178e-06,
+      "loss": 0.0115,
+      "step": 785
+    },
+    {
+      "epoch": 0.9112536947588494,
+      "grad_norm": 0.271484375,
+      "learning_rate": 9.013558189654819e-06,
+      "loss": 0.009,
+      "step": 790
+    },
+    {
+      "epoch": 0.9170211232066903,
+      "grad_norm": 0.216796875,
+      "learning_rate": 8.999586840351676e-06,
+      "loss": 0.0101,
+      "step": 795
+    },
+    {
+      "epoch": 0.922788551654531,
+      "grad_norm": 0.2041015625,
+      "learning_rate": 8.985528234031799e-06,
+      "loss": 0.0163,
+      "step": 800
+    },
+    {
+      "epoch": 0.9285559801023718,
+      "grad_norm": 0.25,
+      "learning_rate": 8.971382677404878e-06,
+      "loss": 0.0094,
+      "step": 805
+    },
+    {
+      "epoch": 0.9343234085502127,
+      "grad_norm": 0.166015625,
+      "learning_rate": 8.957150479077559e-06,
+      "loss": 0.0074,
+      "step": 810
+    },
+    {
+      "epoch": 0.9400908369980535,
+      "grad_norm": 0.189453125,
+      "learning_rate": 8.942831949546702e-06,
+      "loss": 0.0118,
+      "step": 815
+    },
+    {
+      "epoch": 0.9458582654458944,
+      "grad_norm": 0.2041015625,
+      "learning_rate": 8.928427401192618e-06,
+      "loss": 0.0084,
+      "step": 820
+    },
+    {
+      "epoch": 0.9516256938937351,
+      "grad_norm": 0.203125,
+      "learning_rate": 8.913937148272247e-06,
+      "loss": 0.0082,
+      "step": 825
+    },
+    {
+      "epoch": 0.9573931223415759,
+      "grad_norm": 0.259765625,
+      "learning_rate": 8.899361506912298e-06,
+      "loss": 0.0089,
+      "step": 830
+    },
+    {
+      "epoch": 0.9631605507894168,
+      "grad_norm": 0.1865234375,
+      "learning_rate": 8.884700795102365e-06,
+      "loss": 0.0114,
+      "step": 835
+    },
+    {
+      "epoch": 0.9689279792372576,
+      "grad_norm": 0.208984375,
+      "learning_rate": 8.869955332687975e-06,
+      "loss": 0.0084,
+      "step": 840
+    },
+    {
+      "epoch": 0.9746954076850984,
+      "grad_norm": 0.2197265625,
+      "learning_rate": 8.855125441363628e-06,
+      "loss": 0.0104,
+      "step": 845
+    },
+    {
+      "epoch": 0.9804628361329393,
+      "grad_norm": 0.203125,
+      "learning_rate": 8.840211444665754e-06,
+      "loss": 0.0094,
+      "step": 850
+    },
+    {
+      "epoch": 0.98623026458078,
+      "grad_norm": 0.1884765625,
+      "learning_rate": 8.825213667965678e-06,
+      "loss": 0.0094,
+      "step": 855
+    },
+    {
+      "epoch": 0.9919976930286208,
+      "grad_norm": 0.25390625,
+      "learning_rate": 8.810132438462509e-06,
+      "loss": 0.01,
+      "step": 860
+    },
+    {
+      "epoch": 0.9977651214764617,
+      "grad_norm": 0.263671875,
+      "learning_rate": 8.794968085176006e-06,
+      "loss": 0.0106,
+      "step": 865
+    },
+    {
+      "epoch": 1.0034604570687045,
+      "grad_norm": 0.22265625,
+      "learning_rate": 8.779720938939396e-06,
+      "loss": 0.0124,
+      "step": 870
+    },
+    {
+      "epoch": 1.0092278855165453,
+      "grad_norm": 0.1875,
+      "learning_rate": 8.764391332392163e-06,
+      "loss": 0.0088,
+      "step": 875
+    },
+    {
+      "epoch": 1.0149953139643861,
+      "grad_norm": 0.2294921875,
+      "learning_rate": 8.748979599972787e-06,
+      "loss": 0.0085,
+      "step": 880
+    },
+    {
+      "epoch": 1.020762742412227,
+      "grad_norm": 0.1923828125,
+      "learning_rate": 8.73348607791144e-06,
+      "loss": 0.0086,
+      "step": 885
+    },
+    {
+      "epoch": 1.0265301708600678,
+      "grad_norm": 0.23046875,
+      "learning_rate": 8.717911104222668e-06,
+      "loss": 0.0094,
+      "step": 890
+    },
+    {
+      "epoch": 1.0322975993079087,
+      "grad_norm": 0.1767578125,
+      "learning_rate": 8.702255018698e-06,
+      "loss": 0.0085,
+      "step": 895
+    },
+    {
+      "epoch": 1.0380650277557495,
+      "grad_norm": 0.2080078125,
+      "learning_rate": 8.686518162898547e-06,
+      "loss": 0.0087,
+      "step": 900
+    },
+    {
+      "epoch": 1.0438324562035903,
+      "grad_norm": 0.224609375,
+      "learning_rate": 8.670700880147544e-06,
+      "loss": 0.0088,
+      "step": 905
+    },
+    {
+      "epoch": 1.049599884651431,
+      "grad_norm": 0.2041015625,
+      "learning_rate": 8.65480351552286e-06,
+      "loss": 0.0085,
+      "step": 910
+    },
+    {
+      "epoch": 1.0553673130992718,
+      "grad_norm": 0.1767578125,
+      "learning_rate": 8.638826415849469e-06,
+      "loss": 0.0088,
+      "step": 915
+    },
+    {
+      "epoch": 1.0611347415471126,
+      "grad_norm": 0.158203125,
+      "learning_rate": 8.622769929691889e-06,
+      "loss": 0.011,
+      "step": 920
+    },
+    {
+      "epoch": 1.0669021699949535,
+      "grad_norm": 0.181640625,
+      "learning_rate": 8.606634407346575e-06,
+      "loss": 0.009,
+      "step": 925
+    },
+    {
+      "epoch": 1.0726695984427943,
+      "grad_norm": 0.1806640625,
+      "learning_rate": 8.590420200834271e-06,
+      "loss": 0.0086,
+      "step": 930
+    },
+    {
+      "epoch": 1.0784370268906351,
+      "grad_norm": 0.267578125,
+      "learning_rate": 8.574127663892334e-06,
+      "loss": 0.0093,
+      "step": 935
+    },
+    {
+      "epoch": 1.084204455338476,
+      "grad_norm": 0.228515625,
+      "learning_rate": 8.557757151967025e-06,
+      "loss": 0.0083,
+      "step": 940
+    },
+    {
+      "epoch": 1.0899718837863168,
+      "grad_norm": 0.1904296875,
+      "learning_rate": 8.541309022205738e-06,
+      "loss": 0.0094,
+      "step": 945
+    },
+    {
+      "epoch": 1.0957393122341577,
+      "grad_norm": 0.1806640625,
+      "learning_rate": 8.524783633449224e-06,
+      "loss": 0.0073,
+      "step": 950
+    },
+    {
+      "epoch": 1.1015067406819985,
+      "grad_norm": 0.158203125,
+      "learning_rate": 8.508181346223749e-06,
+      "loss": 0.0076,
+      "step": 955
+    },
+    {
+      "epoch": 1.1072741691298393,
+      "grad_norm": 0.173828125,
+      "learning_rate": 8.491502522733243e-06,
+      "loss": 0.0081,
+      "step": 960
+    },
+    {
+      "epoch": 1.11304159757768,
+      "grad_norm": 0.1845703125,
+      "learning_rate": 8.474747526851383e-06,
+      "loss": 0.007,
+      "step": 965
+    },
+    {
+      "epoch": 1.1188090260255208,
+      "grad_norm": 0.2109375,
+      "learning_rate": 8.457916724113667e-06,
+      "loss": 0.0087,
+      "step": 970
+    },
+    {
+      "epoch": 1.1245764544733616,
+      "grad_norm": 0.1513671875,
+      "learning_rate": 8.441010481709424e-06,
+      "loss": 0.0073,
+      "step": 975
+    },
+    {
+      "epoch": 1.1303438829212025,
+      "grad_norm": 0.1884765625,
+      "learning_rate": 8.424029168473829e-06,
+      "loss": 0.009,
+      "step": 980
+    },
+    {
+      "epoch": 1.1361113113690433,
+      "grad_norm": 0.240234375,
+      "learning_rate": 8.406973154879826e-06,
+      "loss": 0.0085,
+      "step": 985
+    },
+    {
+      "epoch": 1.1418787398168841,
+      "grad_norm": 0.150390625,
+      "learning_rate": 8.389842813030065e-06,
+      "loss": 0.0076,
+      "step": 990
+    },
+    {
+      "epoch": 1.147646168264725,
+      "grad_norm": 0.1923828125,
+      "learning_rate": 8.372638516648782e-06,
+      "loss": 0.0094,
+      "step": 995
+    },
+    {
+      "epoch": 1.1534135967125658,
+      "grad_norm": 0.2099609375,
+      "learning_rate": 8.355360641073637e-06,
+      "loss": 0.0086,
+      "step": 1000
+    },
+    {
+      "epoch": 1.1591810251604067,
+      "grad_norm": 0.14453125,
+      "learning_rate": 8.338009563247535e-06,
+      "loss": 0.0084,
+      "step": 1005
+    },
+    {
+      "epoch": 1.1649484536082475,
+      "grad_norm": 0.197265625,
+      "learning_rate": 8.3205856617104e-06,
+      "loss": 0.0111,
+      "step": 1010
+    },
+    {
+      "epoch": 1.1707158820560881,
+      "grad_norm": 0.2109375,
+      "learning_rate": 8.30308931659091e-06,
+      "loss": 0.0086,
+      "step": 1015
+    },
+    {
+      "epoch": 1.176483310503929,
+      "grad_norm": 0.1982421875,
+      "learning_rate": 8.285520909598213e-06,
+      "loss": 0.0078,
+      "step": 1020
+    },
+    {
+      "epoch": 1.1822507389517698,
+      "grad_norm": 0.203125,
+      "learning_rate": 8.267880824013591e-06,
+      "loss": 0.0082,
+      "step": 1025
+    },
+    {
+      "epoch": 1.1880181673996106,
+      "grad_norm": 0.201171875,
+      "learning_rate": 8.250169444682109e-06,
+      "loss": 0.0084,
+      "step": 1030
+    },
+    {
+      "epoch": 1.1937855958474515,
+      "grad_norm": 0.2158203125,
+      "learning_rate": 8.232387158004205e-06,
+      "loss": 0.0071,
+      "step": 1035
+    },
+    {
+      "epoch": 1.1995530242952923,
+      "grad_norm": 0.224609375,
+      "learning_rate": 8.214534351927275e-06,
+      "loss": 0.0096,
+      "step": 1040
+    },
+    {
+      "epoch": 1.2053204527431332,
+      "grad_norm": 0.1806640625,
+      "learning_rate": 8.196611415937196e-06,
+      "loss": 0.0083,
+      "step": 1045
+    },
+    {
+      "epoch": 1.211087881190974,
+      "grad_norm": 0.201171875,
+      "learning_rate": 8.178618741049841e-06,
+      "loss": 0.0101,
+      "step": 1050
+    },
+    {
+      "epoch": 1.2168553096388148,
+      "grad_norm": 0.25390625,
+      "learning_rate": 8.160556719802535e-06,
+      "loss": 0.0078,
+      "step": 1055
+    },
+    {
+      "epoch": 1.2226227380866557,
+      "grad_norm": 0.16015625,
+      "learning_rate": 8.142425746245503e-06,
+      "loss": 0.0078,
+      "step": 1060
+    },
+    {
+      "epoch": 1.2283901665344965,
+      "grad_norm": 0.1826171875,
+      "learning_rate": 8.124226215933268e-06,
+      "loss": 0.0101,
+      "step": 1065
+    },
+    {
+      "epoch": 1.2341575949823373,
+      "grad_norm": 0.2470703125,
+      "learning_rate": 8.10595852591602e-06,
+      "loss": 0.008,
+      "step": 1070
+    },
+    {
+      "epoch": 1.2399250234301782,
+      "grad_norm": 0.25,
+      "learning_rate": 8.08762307473096e-06,
+      "loss": 0.0075,
+      "step": 1075
+    },
+    {
+      "epoch": 1.2456924518780188,
+      "grad_norm": 0.2080078125,
+      "learning_rate": 8.069220262393597e-06,
+      "loss": 0.0105,
+      "step": 1080
+    },
+    {
+      "epoch": 1.2514598803258596,
+      "grad_norm": 0.2294921875,
+      "learning_rate": 8.050750490389026e-06,
+      "loss": 0.0076,
+      "step": 1085
+    },
+    {
+      "epoch": 1.2572273087737005,
+      "grad_norm": 0.173828125,
+      "learning_rate": 8.03221416166317e-06,
+      "loss": 0.0083,
+      "step": 1090
+    },
+    {
+      "epoch": 1.2629947372215413,
+      "grad_norm": 0.2080078125,
+      "learning_rate": 8.013611680613983e-06,
+      "loss": 0.0082,
+      "step": 1095
+    },
+    {
+      "epoch": 1.2687621656693822,
+      "grad_norm": 0.1953125,
+      "learning_rate": 7.994943453082636e-06,
+      "loss": 0.0097,
+      "step": 1100
+    },
+    {
+      "epoch": 1.274529594117223,
+      "grad_norm": 0.193359375,
+      "learning_rate": 7.976209886344654e-06,
+      "loss": 0.0087,
+      "step": 1105
+    },
+    {
+      "epoch": 1.2802970225650638,
+      "grad_norm": 0.197265625,
+      "learning_rate": 7.957411389101045e-06,
+      "loss": 0.0077,
+      "step": 1110
+    },
+    {
+      "epoch": 1.2860644510129047,
+      "grad_norm": 0.1953125,
+      "learning_rate": 7.938548371469362e-06,
+      "loss": 0.01,
+      "step": 1115
+    },
+    {
+      "epoch": 1.2918318794607455,
+      "grad_norm": 0.162109375,
+      "learning_rate": 7.919621244974773e-06,
+      "loss": 0.0189,
+      "step": 1120
+    },
+    {
+      "epoch": 1.2975993079085861,
+      "grad_norm": 0.296875,
+      "learning_rate": 7.900630422541079e-06,
+      "loss": 0.0106,
+      "step": 1125
+    },
+    {
+      "epoch": 1.303366736356427,
+      "grad_norm": 0.1728515625,
+      "learning_rate": 7.881576318481702e-06,
+      "loss": 0.0085,
+      "step": 1130
+    },
+    {
+      "epoch": 1.3091341648042678,
+      "grad_norm": 0.2177734375,
+      "learning_rate": 7.862459348490645e-06,
+      "loss": 0.0095,
+      "step": 1135
+    },
+    {
+      "epoch": 1.3149015932521086,
+      "grad_norm": 0.158203125,
+      "learning_rate": 7.843279929633434e-06,
+      "loss": 0.009,
+      "step": 1140
+    },
+    {
+      "epoch": 1.3206690216999495,
+      "grad_norm": 0.1767578125,
+      "learning_rate": 7.824038480338002e-06,
+      "loss": 0.0078,
+      "step": 1145
+    },
+    {
+      "epoch": 1.3264364501477903,
+      "grad_norm": 0.193359375,
+      "learning_rate": 7.804735420385578e-06,
+      "loss": 0.0091,
+      "step": 1150
+    },
+    {
+      "epoch": 1.3322038785956312,
+      "grad_norm": 0.2421875,
+      "learning_rate": 7.785371170901512e-06,
+      "loss": 0.0084,
+      "step": 1155
+    },
+    {
+      "epoch": 1.337971307043472,
+      "grad_norm": 0.181640625,
+      "learning_rate": 7.765946154346105e-06,
+      "loss": 0.012,
+      "step": 1160
+    },
+    {
+      "epoch": 1.3437387354913128,
+      "grad_norm": 0.1435546875,
+      "learning_rate": 7.746460794505375e-06,
+      "loss": 0.0082,
+      "step": 1165
+    },
+    {
+      "epoch": 1.3495061639391537,
+      "grad_norm": 0.203125,
+      "learning_rate": 7.726915516481824e-06,
+      "loss": 0.0079,
+      "step": 1170
+    },
+    {
+      "epoch": 1.3552735923869945,
+      "grad_norm": 0.1796875,
+      "learning_rate": 7.70731074668516e-06,
+      "loss": 0.0077,
+      "step": 1175
+    },
+    {
+      "epoch": 1.3610410208348354,
+      "grad_norm": 0.1494140625,
+      "learning_rate": 7.68764691282299e-06,
+      "loss": 0.0079,
+      "step": 1180
+    },
+    {
+      "epoch": 1.3668084492826762,
+      "grad_norm": 0.1455078125,
+      "learning_rate": 7.667924443891498e-06,
+      "loss": 0.0086,
+      "step": 1185
+    },
+    {
+      "epoch": 1.372575877730517,
+      "grad_norm": 0.193359375,
+      "learning_rate": 7.648143770166074e-06,
+      "loss": 0.0075,
+      "step": 1190
+    },
+    {
+      "epoch": 1.3783433061783577,
+      "grad_norm": 0.16796875,
+      "learning_rate": 7.628305323191942e-06,
+      "loss": 0.0085,
+      "step": 1195
+    },
+    {
+      "epoch": 1.3841107346261985,
+      "grad_norm": 0.2099609375,
+      "learning_rate": 7.608409535774725e-06,
+      "loss": 0.0081,
+      "step": 1200
+    },
+    {
+      "epoch": 1.3898781630740393,
+      "grad_norm": 0.11962890625,
+      "learning_rate": 7.588456841971026e-06,
+      "loss": 0.0078,
+      "step": 1205
+    },
+    {
+      "epoch": 1.3956455915218802,
+      "grad_norm": 0.1494140625,
+      "learning_rate": 7.568447677078937e-06,
+      "loss": 0.0085,
+      "step": 1210
+    },
+    {
+      "epoch": 1.401413019969721,
+      "grad_norm": 0.20703125,
+      "learning_rate": 7.54838247762856e-06,
+      "loss": 0.0074,
+      "step": 1215
+    },
+    {
+      "epoch": 1.4071804484175618,
+      "grad_norm": 0.2119140625,
+      "learning_rate": 7.528261681372468e-06,
+      "loss": 0.0072,
+      "step": 1220
+    },
+    {
+      "epoch": 1.4129478768654027,
+      "grad_norm": 0.1982421875,
+      "learning_rate": 7.508085727276169e-06,
+      "loss": 0.0095,
+      "step": 1225
+    },
+    {
+      "epoch": 1.4187153053132435,
+      "grad_norm": 0.2294921875,
+      "learning_rate": 7.487855055508516e-06,
+      "loss": 0.0083,
+      "step": 1230
+    },
+    {
+      "epoch": 1.4244827337610841,
+      "grad_norm": 0.125,
+      "learning_rate": 7.467570107432115e-06,
+      "loss": 0.0073,
+      "step": 1235
+    },
+    {
+      "epoch": 1.430250162208925,
+      "grad_norm": 0.2001953125,
+      "learning_rate": 7.447231325593689e-06,
+      "loss": 0.0085,
+      "step": 1240
+    },
+    {
+      "epoch": 1.4360175906567658,
+      "grad_norm": 0.1708984375,
+      "learning_rate": 7.426839153714429e-06,
+      "loss": 0.0084,
+      "step": 1245
+    },
+    {
+      "epoch": 1.4417850191046067,
+      "grad_norm": 0.19921875,
+      "learning_rate": 7.406394036680306e-06,
+      "loss": 0.0071,
+      "step": 1250
+    },
+    {
+      "epoch": 1.4475524475524475,
+      "grad_norm": 0.2294921875,
+      "learning_rate": 7.385896420532372e-06,
+      "loss": 0.0086,
+      "step": 1255
+    },
+    {
+      "epoch": 1.4533198760002883,
+      "grad_norm": 0.2392578125,
+      "learning_rate": 7.365346752457026e-06,
+      "loss": 0.0095,
+      "step": 1260
+    },
+    {
+      "epoch": 1.4590873044481292,
+      "grad_norm": 0.1494140625,
+      "learning_rate": 7.3447454807762565e-06,
+      "loss": 0.0073,
+      "step": 1265
+    },
+    {
+      "epoch": 1.46485473289597,
+      "grad_norm": 0.1787109375,
+      "learning_rate": 7.324093054937864e-06,
+      "loss": 0.0088,
+      "step": 1270
+    },
+    {
+      "epoch": 1.4706221613438109,
+      "grad_norm": 0.2060546875,
+      "learning_rate": 7.3033899255056575e-06,
+      "loss": 0.0093,
+      "step": 1275
+    },
+    {
+      "epoch": 1.4763895897916517,
+      "grad_norm": 0.2353515625,
+      "learning_rate": 7.282636544149613e-06,
+      "loss": 0.0094,
+      "step": 1280
+    },
+    {
+      "epoch": 1.4821570182394925,
+      "grad_norm": 0.1982421875,
+      "learning_rate": 7.261833363636036e-06,
+      "loss": 0.0083,
+      "step": 1285
+    },
+    {
+      "epoch": 1.4879244466873334,
+      "grad_norm": 0.2578125,
+      "learning_rate": 7.24098083781767e-06,
+      "loss": 0.01,
+      "step": 1290
+    },
+    {
+      "epoch": 1.4936918751351742,
+      "grad_norm": 0.154296875,
+      "learning_rate": 7.220079421623805e-06,
+      "loss": 0.0074,
+      "step": 1295
+    },
+    {
+      "epoch": 1.499459303583015,
+      "grad_norm": 0.189453125,
+      "learning_rate": 7.199129571050345e-06,
+      "loss": 0.0115,
+      "step": 1300
+    },
+    {
+      "epoch": 1.5052267320308559,
+      "grad_norm": 0.1611328125,
+      "learning_rate": 7.178131743149868e-06,
+      "loss": 0.0076,
+      "step": 1305
+    },
+    {
+      "epoch": 1.5109941604786967,
+      "grad_norm": 0.1650390625,
+      "learning_rate": 7.157086396021641e-06,
+      "loss": 0.0083,
+      "step": 1310
+    },
+    {
+      "epoch": 1.5167615889265373,
+      "grad_norm": 0.267578125,
+      "learning_rate": 7.135993988801644e-06,
+      "loss": 0.0084,
+      "step": 1315
+    },
+    {
+      "epoch": 1.5225290173743782,
+      "grad_norm": 0.1943359375,
+      "learning_rate": 7.114854981652537e-06,
+      "loss": 0.0072,
+      "step": 1320
+    },
+    {
+      "epoch": 1.528296445822219,
+      "grad_norm": 0.275390625,
+      "learning_rate": 7.093669835753632e-06,
+      "loss": 0.007,
+      "step": 1325
+    },
+    {
+      "epoch": 1.5340638742700599,
+      "grad_norm": 0.2451171875,
+      "learning_rate": 7.072439013290824e-06,
+      "loss": 0.0084,
+      "step": 1330
+    },
+    {
+      "epoch": 1.5398313027179007,
+      "grad_norm": 0.2353515625,
+      "learning_rate": 7.051162977446513e-06,
+      "loss": 0.0074,
+      "step": 1335
+    },
+    {
+      "epoch": 1.5455987311657413,
+      "grad_norm": 0.11376953125,
+      "learning_rate": 7.029842192389498e-06,
+      "loss": 0.0097,
+      "step": 1340
+    },
+    {
+      "epoch": 1.5513661596135822,
+      "grad_norm": 0.2421875,
+      "learning_rate": 7.008477123264849e-06,
+      "loss": 0.0086,
+      "step": 1345
+    },
+    {
+      "epoch": 1.557133588061423,
+      "grad_norm": 0.201171875,
+      "learning_rate": 6.987068236183756e-06,
+      "loss": 0.0096,
+      "step": 1350
+    },
+    {
+      "epoch": 1.5629010165092638,
+      "grad_norm": 0.29296875,
+      "learning_rate": 6.965615998213371e-06,
+      "loss": 0.0083,
+      "step": 1355
+    },
+    {
+      "epoch": 1.5686684449571047,
+      "grad_norm": 0.205078125,
+      "learning_rate": 6.944120877366605e-06,
+      "loss": 0.0081,
+      "step": 1360
+    },
+    {
+      "epoch": 1.5744358734049455,
+      "grad_norm": 0.1513671875,
+      "learning_rate": 6.922583342591924e-06,
+      "loss": 0.0083,
+      "step": 1365
+    },
+    {
+      "epoch": 1.5802033018527863,
+      "grad_norm": 0.2109375,
+      "learning_rate": 6.901003863763121e-06,
+      "loss": 0.0076,
+      "step": 1370
+    },
+    {
+      "epoch": 1.5859707303006272,
+      "grad_norm": 0.1865234375,
+      "learning_rate": 6.87938291166906e-06,
+      "loss": 0.0072,
+      "step": 1375
+    },
+    {
+      "epoch": 1.591738158748468,
+      "grad_norm": 0.142578125,
+      "learning_rate": 6.857720958003406e-06,
+      "loss": 0.0073,
+      "step": 1380
+    },
+    {
+      "epoch": 1.5975055871963089,
+      "grad_norm": 0.2470703125,
+      "learning_rate": 6.836018475354339e-06,
+      "loss": 0.0114,
+      "step": 1385
+    },
+    {
+      "epoch": 1.6032730156441497,
+      "grad_norm": 0.1845703125,
+      "learning_rate": 6.814275937194233e-06,
+      "loss": 0.007,
+      "step": 1390
+    },
+    {
+      "epoch": 1.6090404440919905,
+      "grad_norm": 0.1474609375,
+      "learning_rate": 6.792493817869339e-06,
+      "loss": 0.007,
+      "step": 1395
+    },
+    {
+      "epoch": 1.6148078725398314,
+      "grad_norm": 0.197265625,
+      "learning_rate": 6.7706725925894264e-06,
+      "loss": 0.0083,
+      "step": 1400
+    },
+    {
+      "epoch": 1.6205753009876722,
+      "grad_norm": 0.1962890625,
+      "learning_rate": 6.748812737417428e-06,
+      "loss": 0.0093,
+      "step": 1405
+    },
+    {
+      "epoch": 1.626342729435513,
+      "grad_norm": 0.2060546875,
+      "learning_rate": 6.726914729259034e-06,
+      "loss": 0.0107,
+      "step": 1410
+    },
+    {
+      "epoch": 1.632110157883354,
+      "grad_norm": 0.2021484375,
+      "learning_rate": 6.704979045852311e-06,
+      "loss": 0.0062,
+      "step": 1415
+    },
+    {
+      "epoch": 1.6378775863311947,
+      "grad_norm": 0.1650390625,
+      "learning_rate": 6.683006165757262e-06,
+      "loss": 0.0075,
+      "step": 1420
+    },
+    {
+      "epoch": 1.6436450147790354,
+      "grad_norm": 0.1640625,
+      "learning_rate": 6.660996568345392e-06,
+      "loss": 0.0105,
+      "step": 1425
+    },
+    {
+      "epoch": 1.6494124432268762,
+      "grad_norm": 0.1396484375,
+      "learning_rate": 6.63895073378925e-06,
+      "loss": 0.0061,
+      "step": 1430
+    },
+    {
+      "epoch": 1.655179871674717,
+      "grad_norm": 0.1728515625,
+      "learning_rate": 6.6168691430519524e-06,
+      "loss": 0.0084,
+      "step": 1435
+    },
+    {
+      "epoch": 1.6609473001225579,
+      "grad_norm": 0.142578125,
+      "learning_rate": 6.594752277876695e-06,
+      "loss": 0.0074,
+      "step": 1440
+    },
+    {
+      "epoch": 1.6667147285703987,
+      "grad_norm": 0.177734375,
+      "learning_rate": 6.572600620776232e-06,
+      "loss": 0.0072,
+      "step": 1445
+    },
+    {
+      "epoch": 1.6724821570182395,
+      "grad_norm": 0.1328125,
+      "learning_rate": 6.55041465502236e-06,
+      "loss": 0.0064,
+      "step": 1450
+    },
+    {
+      "epoch": 1.6782495854660802,
+      "grad_norm": 0.2021484375,
+      "learning_rate": 6.52819486463537e-06,
+      "loss": 0.0077,
+      "step": 1455
+    },
+    {
+      "epoch": 1.684017013913921,
+      "grad_norm": 0.18359375,
+      "learning_rate": 6.505941734373489e-06,
+      "loss": 0.0078,
+      "step": 1460
+    },
+    {
+      "epoch": 1.6897844423617618,
+      "grad_norm": 0.197265625,
+      "learning_rate": 6.4836557497222995e-06,
+      "loss": 0.007,
+      "step": 1465
+    },
+    {
+      "epoch": 1.6955518708096027,
+      "grad_norm": 0.181640625,
+      "learning_rate": 6.461337396884158e-06,
+      "loss": 0.0088,
+      "step": 1470
+    },
+    {
+      "epoch": 1.7013192992574435,
+      "grad_norm": 0.1923828125,
+      "learning_rate": 6.438987162767578e-06,
+      "loss": 0.0151,
+      "step": 1475
+    },
+    {
+      "epoch": 1.7070867277052844,
+      "grad_norm": 0.1591796875,
+      "learning_rate": 6.416605534976614e-06,
+      "loss": 0.007,
+      "step": 1480
+    },
+    {
+      "epoch": 1.7128541561531252,
+      "grad_norm": 0.125,
+      "learning_rate": 6.3941930018002175e-06,
+      "loss": 0.0075,
+      "step": 1485
+    },
+    {
+      "epoch": 1.718621584600966,
+      "grad_norm": 0.2353515625,
+      "learning_rate": 6.371750052201588e-06,
+      "loss": 0.0085,
+      "step": 1490
+    },
+    {
+      "epoch": 1.7243890130488069,
+      "grad_norm": 0.1435546875,
+      "learning_rate": 6.349277175807506e-06,
+      "loss": 0.0074,
+      "step": 1495
+    },
+    {
+      "epoch": 1.7301564414966477,
+      "grad_norm": 0.169921875,
+      "learning_rate": 6.326774862897653e-06,
+      "loss": 0.0069,
+      "step": 1500
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 3468,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 4,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.2204021726513725e+18,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoints/sft3b_lr1e5_bs64/checkpoint-2000/added_tokens.json

@@ -0,0 +1,35 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|action_sep|>": 151670,
+  "<|arg_sep|>": 151671,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|call_sep|>": 151666,
+  "<|end_of_text|>": 151673,
+  "<|endoftext|>": 151643,
+  "<|exception_sep|>": 151669,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|frame_sep|>": 151672,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|latent_end|>": 151675,
+  "<|latent_start|>": 151674,
+  "<|line_sep|>": 151667,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|return_sep|>": 151668,
+  "<|trace_context_start|>": 151665,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoints/sft3b_lr1e5_bs64/checkpoint-2000/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

checkpoints/sft3b_lr1e5_bs64/checkpoint-2000/config.json

@@ -0,0 +1,66 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "dtype": "bfloat16",
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 36,
+  "model_type": "qwen2",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 36,
+  "num_key_value_heads": 2,
+  "pad_token_id": 151643,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "transformers_version": "4.57.6",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151676
+}

checkpoints/sft3b_lr1e5_bs64/checkpoint-2000/generation_config.json

@@ -0,0 +1,8 @@
+{
+  "eos_token_id": [
+    151643
+  ],
+  "max_new_tokens": 2048,
+  "pad_token_id": 151643,
+  "transformers_version": "4.57.6"
+}

checkpoints/sft3b_lr1e5_bs64/checkpoint-2000/model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a26004ce73a62ea69921682dce061af46ce9ca8d0ea0a4014ac5965a9205b41
+size 4956495344

checkpoints/sft3b_lr1e5_bs64/checkpoint-2000/model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76daf6328f5c81b026ec87cca127821f9da2b67bb79a7ef4087aabb8bb0fdd1f
+size 1214366696

checkpoints/sft3b_lr1e5_bs64/checkpoint-2000/model.safetensors.index.json

@@ -0,0 +1,442 @@
+{
+  "metadata": {
+    "total_parameters": 3085406208,
+    "total_size": 6170812416
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.28.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.28.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

checkpoints/sft3b_lr1e5_bs64/checkpoint-2000/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoints/sft3b_lr1e5_bs64/checkpoint-2000/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83a790d654474f5dfe225f889afd0210313eb1083f942671f2c4b8e95a1c922b
+size 11424004

checkpoints/sft3b_lr1e5_bs64/checkpoint-2000/tokenizer_config.json

@@ -0,0 +1,295 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<|trace_context_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151666": {
+      "content": "<|call_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151667": {
+      "content": "<|line_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151668": {
+      "content": "<|return_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151669": {
+      "content": "<|exception_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151670": {
+      "content": "<|action_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151671": {
+      "content": "<|arg_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151672": {
+      "content": "<|frame_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151673": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151674": {
+      "content": "<|latent_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151675": {
+      "content": "<|latent_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoints/sft3b_lr1e5_bs64/checkpoint-2000/trainer_state.json

@@ -0,0 +1,2834 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.3068271934251317,
+  "eval_steps": 500,
+  "global_step": 2000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.005767428447840819,
+      "grad_norm": 27.875,
+      "learning_rate": 3.80952380952381e-07,
+      "loss": 1.3844,
+      "step": 5
+    },
+    {
+      "epoch": 0.011534856895681638,
+      "grad_norm": 27.75,
+      "learning_rate": 8.571428571428572e-07,
+      "loss": 1.4373,
+      "step": 10
+    },
+    {
+      "epoch": 0.017302285343522458,
+      "grad_norm": 28.625,
+      "learning_rate": 1.3333333333333334e-06,
+      "loss": 1.4472,
+      "step": 15
+    },
+    {
+      "epoch": 0.023069713791363276,
+      "grad_norm": 27.5,
+      "learning_rate": 1.8095238095238097e-06,
+      "loss": 1.4299,
+      "step": 20
+    },
+    {
+      "epoch": 0.028837142239204094,
+      "grad_norm": 21.5,
+      "learning_rate": 2.285714285714286e-06,
+      "loss": 1.3611,
+      "step": 25
+    },
+    {
+      "epoch": 0.034604570687044915,
+      "grad_norm": 14.375,
+      "learning_rate": 2.7619047619047625e-06,
+      "loss": 1.2402,
+      "step": 30
+    },
+    {
+      "epoch": 0.04037199913488573,
+      "grad_norm": 12.8125,
+      "learning_rate": 3.2380952380952385e-06,
+      "loss": 1.1743,
+      "step": 35
+    },
+    {
+      "epoch": 0.04613942758272655,
+      "grad_norm": 11.9375,
+      "learning_rate": 3.7142857142857146e-06,
+      "loss": 1.1619,
+      "step": 40
+    },
+    {
+      "epoch": 0.05190685603056737,
+      "grad_norm": 10.3125,
+      "learning_rate": 4.190476190476191e-06,
+      "loss": 1.0707,
+      "step": 45
+    },
+    {
+      "epoch": 0.05767428447840819,
+      "grad_norm": 9.875,
+      "learning_rate": 4.666666666666667e-06,
+      "loss": 0.9887,
+      "step": 50
+    },
+    {
+      "epoch": 0.063441712926249,
+      "grad_norm": 10.25,
+      "learning_rate": 5.142857142857142e-06,
+      "loss": 0.9292,
+      "step": 55
+    },
+    {
+      "epoch": 0.06920914137408983,
+      "grad_norm": 10.8125,
+      "learning_rate": 5.619047619047619e-06,
+      "loss": 0.8973,
+      "step": 60
+    },
+    {
+      "epoch": 0.07497656982193064,
+      "grad_norm": 9.8125,
+      "learning_rate": 6.095238095238096e-06,
+      "loss": 0.8068,
+      "step": 65
+    },
+    {
+      "epoch": 0.08074399826977147,
+      "grad_norm": 10.75,
+      "learning_rate": 6.571428571428572e-06,
+      "loss": 0.8352,
+      "step": 70
+    },
+    {
+      "epoch": 0.08651142671761229,
+      "grad_norm": 10.0625,
+      "learning_rate": 7.047619047619048e-06,
+      "loss": 0.7974,
+      "step": 75
+    },
+    {
+      "epoch": 0.0922788551654531,
+      "grad_norm": 10.5,
+      "learning_rate": 7.523809523809524e-06,
+      "loss": 0.7557,
+      "step": 80
+    },
+    {
+      "epoch": 0.09804628361329393,
+      "grad_norm": 9.9375,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 0.7018,
+      "step": 85
+    },
+    {
+      "epoch": 0.10381371206113474,
+      "grad_norm": 9.4375,
+      "learning_rate": 8.476190476190477e-06,
+      "loss": 0.688,
+      "step": 90
+    },
+    {
+      "epoch": 0.10958114050897556,
+      "grad_norm": 9.4375,
+      "learning_rate": 8.952380952380953e-06,
+      "loss": 0.6379,
+      "step": 95
+    },
+    {
+      "epoch": 0.11534856895681637,
+      "grad_norm": 8.875,
+      "learning_rate": 9.42857142857143e-06,
+      "loss": 0.5856,
+      "step": 100
+    },
+    {
+      "epoch": 0.1211159974046572,
+      "grad_norm": 9.125,
+      "learning_rate": 9.904761904761906e-06,
+      "loss": 0.5049,
+      "step": 105
+    },
+    {
+      "epoch": 0.126883425852498,
+      "grad_norm": 8.4375,
+      "learning_rate": 9.99996509356498e-06,
+      "loss": 0.4436,
+      "step": 110
+    },
+    {
+      "epoch": 0.13265085430033885,
+      "grad_norm": 7.0,
+      "learning_rate": 9.999823287008022e-06,
+      "loss": 0.3535,
+      "step": 115
+    },
+    {
+      "epoch": 0.13841828274817966,
+      "grad_norm": 5.96875,
+      "learning_rate": 9.999572401768278e-06,
+      "loss": 0.2817,
+      "step": 120
+    },
+    {
+      "epoch": 0.14418571119602047,
+      "grad_norm": 4.625,
+      "learning_rate": 9.999212443319191e-06,
+      "loss": 0.2508,
+      "step": 125
+    },
+    {
+      "epoch": 0.14995313964386128,
+      "grad_norm": 3.5,
+      "learning_rate": 9.998743419513795e-06,
+      "loss": 0.2178,
+      "step": 130
+    },
+    {
+      "epoch": 0.15572056809170212,
+      "grad_norm": 2.21875,
+      "learning_rate": 9.998165340584553e-06,
+      "loss": 0.1906,
+      "step": 135
+    },
+    {
+      "epoch": 0.16148799653954293,
+      "grad_norm": 1.59375,
+      "learning_rate": 9.99747821914313e-06,
+      "loss": 0.1707,
+      "step": 140
+    },
+    {
+      "epoch": 0.16725542498738374,
+      "grad_norm": 1.78125,
+      "learning_rate": 9.99668207018011e-06,
+      "loss": 0.1657,
+      "step": 145
+    },
+    {
+      "epoch": 0.17302285343522458,
+      "grad_norm": 1.71875,
+      "learning_rate": 9.99577691106469e-06,
+      "loss": 0.1475,
+      "step": 150
+    },
+    {
+      "epoch": 0.1787902818830654,
+      "grad_norm": 1.703125,
+      "learning_rate": 9.994762761544276e-06,
+      "loss": 0.139,
+      "step": 155
+    },
+    {
+      "epoch": 0.1845577103309062,
+      "grad_norm": 1.8125,
+      "learning_rate": 9.993639643744071e-06,
+      "loss": 0.1209,
+      "step": 160
+    },
+    {
+      "epoch": 0.19032513877874702,
+      "grad_norm": 1.828125,
+      "learning_rate": 9.992407582166582e-06,
+      "loss": 0.1078,
+      "step": 165
+    },
+    {
+      "epoch": 0.19609256722658785,
+      "grad_norm": 1.8046875,
+      "learning_rate": 9.991066603691092e-06,
+      "loss": 0.0954,
+      "step": 170
+    },
+    {
+      "epoch": 0.20185999567442867,
+      "grad_norm": 1.6484375,
+      "learning_rate": 9.989616737573064e-06,
+      "loss": 0.0848,
+      "step": 175
+    },
+    {
+      "epoch": 0.20762742412226948,
+      "grad_norm": 1.296875,
+      "learning_rate": 9.988058015443517e-06,
+      "loss": 0.0691,
+      "step": 180
+    },
+    {
+      "epoch": 0.2133948525701103,
+      "grad_norm": 1.0234375,
+      "learning_rate": 9.98639047130832e-06,
+      "loss": 0.0602,
+      "step": 185
+    },
+    {
+      "epoch": 0.21916228101795113,
+      "grad_norm": 0.96484375,
+      "learning_rate": 9.984614141547468e-06,
+      "loss": 0.0522,
+      "step": 190
+    },
+    {
+      "epoch": 0.22492970946579194,
+      "grad_norm": 0.77734375,
+      "learning_rate": 9.98272906491427e-06,
+      "loss": 0.0472,
+      "step": 195
+    },
+    {
+      "epoch": 0.23069713791363275,
+      "grad_norm": 0.6328125,
+      "learning_rate": 9.980735282534512e-06,
+      "loss": 0.046,
+      "step": 200
+    },
+    {
+      "epoch": 0.2364645663614736,
+      "grad_norm": 0.7109375,
+      "learning_rate": 9.978632837905566e-06,
+      "loss": 0.0327,
+      "step": 205
+    },
+    {
+      "epoch": 0.2422319948093144,
+      "grad_norm": 0.58984375,
+      "learning_rate": 9.976421776895432e-06,
+      "loss": 0.0341,
+      "step": 210
+    },
+    {
+      "epoch": 0.2479994232571552,
+      "grad_norm": 0.50390625,
+      "learning_rate": 9.974102147741735e-06,
+      "loss": 0.0311,
+      "step": 215
+    },
+    {
+      "epoch": 0.253766851704996,
+      "grad_norm": 0.47265625,
+      "learning_rate": 9.971674001050687e-06,
+      "loss": 0.0284,
+      "step": 220
+    },
+    {
+      "epoch": 0.25953428015283686,
+      "grad_norm": 0.41796875,
+      "learning_rate": 9.969137389795964e-06,
+      "loss": 0.0262,
+      "step": 225
+    },
+    {
+      "epoch": 0.2653017086006777,
+      "grad_norm": 0.5,
+      "learning_rate": 9.966492369317568e-06,
+      "loss": 0.0246,
+      "step": 230
+    },
+    {
+      "epoch": 0.2710691370485185,
+      "grad_norm": 0.40234375,
+      "learning_rate": 9.963738997320609e-06,
+      "loss": 0.0223,
+      "step": 235
+    },
+    {
+      "epoch": 0.2768365654963593,
+      "grad_norm": 0.43359375,
+      "learning_rate": 9.960877333874045e-06,
+      "loss": 0.0237,
+      "step": 240
+    },
+    {
+      "epoch": 0.2826039939442001,
+      "grad_norm": 0.4140625,
+      "learning_rate": 9.95790744140938e-06,
+      "loss": 0.0292,
+      "step": 245
+    },
+    {
+      "epoch": 0.28837142239204094,
+      "grad_norm": 0.39453125,
+      "learning_rate": 9.954829384719296e-06,
+      "loss": 0.0229,
+      "step": 250
+    },
+    {
+      "epoch": 0.2941388508398818,
+      "grad_norm": 0.35546875,
+      "learning_rate": 9.95164323095624e-06,
+      "loss": 0.0198,
+      "step": 255
+    },
+    {
+      "epoch": 0.29990627928772257,
+      "grad_norm": 0.427734375,
+      "learning_rate": 9.948349049630959e-06,
+      "loss": 0.0181,
+      "step": 260
+    },
+    {
+      "epoch": 0.3056737077355634,
+      "grad_norm": 0.35546875,
+      "learning_rate": 9.944946912610986e-06,
+      "loss": 0.0207,
+      "step": 265
+    },
+    {
+      "epoch": 0.31144113618340424,
+      "grad_norm": 0.33984375,
+      "learning_rate": 9.941436894119068e-06,
+      "loss": 0.017,
+      "step": 270
+    },
+    {
+      "epoch": 0.317208564631245,
+      "grad_norm": 0.416015625,
+      "learning_rate": 9.937819070731551e-06,
+      "loss": 0.0157,
+      "step": 275
+    },
+    {
+      "epoch": 0.32297599307908587,
+      "grad_norm": 0.337890625,
+      "learning_rate": 9.934093521376707e-06,
+      "loss": 0.0177,
+      "step": 280
+    },
+    {
+      "epoch": 0.3287434215269267,
+      "grad_norm": 0.328125,
+      "learning_rate": 9.93026032733301e-06,
+      "loss": 0.0164,
+      "step": 285
+    },
+    {
+      "epoch": 0.3345108499747675,
+      "grad_norm": 0.30859375,
+      "learning_rate": 9.926319572227369e-06,
+      "loss": 0.0149,
+      "step": 290
+    },
+    {
+      "epoch": 0.3402782784226083,
+      "grad_norm": 0.31640625,
+      "learning_rate": 9.922271342033295e-06,
+      "loss": 0.0194,
+      "step": 295
+    },
+    {
+      "epoch": 0.34604570687044917,
+      "grad_norm": 0.3515625,
+      "learning_rate": 9.918115725069038e-06,
+      "loss": 0.0152,
+      "step": 300
+    },
+    {
+      "epoch": 0.35181313531828995,
+      "grad_norm": 0.287109375,
+      "learning_rate": 9.913852811995642e-06,
+      "loss": 0.0163,
+      "step": 305
+    },
+    {
+      "epoch": 0.3575805637661308,
+      "grad_norm": 0.37890625,
+      "learning_rate": 9.909482695814986e-06,
+      "loss": 0.0138,
+      "step": 310
+    },
+    {
+      "epoch": 0.36334799221397157,
+      "grad_norm": 0.31640625,
+      "learning_rate": 9.90500547186774e-06,
+      "loss": 0.0174,
+      "step": 315
+    },
+    {
+      "epoch": 0.3691154206618124,
+      "grad_norm": 0.337890625,
+      "learning_rate": 9.900421237831293e-06,
+      "loss": 0.0154,
+      "step": 320
+    },
+    {
+      "epoch": 0.37488284910965325,
+      "grad_norm": 0.2578125,
+      "learning_rate": 9.895730093717629e-06,
+      "loss": 0.0169,
+      "step": 325
+    },
+    {
+      "epoch": 0.38065027755749403,
+      "grad_norm": 0.3359375,
+      "learning_rate": 9.890932141871122e-06,
+      "loss": 0.0151,
+      "step": 330
+    },
+    {
+      "epoch": 0.38641770600533487,
+      "grad_norm": 0.251953125,
+      "learning_rate": 9.886027486966326e-06,
+      "loss": 0.0146,
+      "step": 335
+    },
+    {
+      "epoch": 0.3921851344531757,
+      "grad_norm": 0.314453125,
+      "learning_rate": 9.881016236005686e-06,
+      "loss": 0.0145,
+      "step": 340
+    },
+    {
+      "epoch": 0.3979525629010165,
+      "grad_norm": 0.22265625,
+      "learning_rate": 9.875898498317194e-06,
+      "loss": 0.0136,
+      "step": 345
+    },
+    {
+      "epoch": 0.40371999134885733,
+      "grad_norm": 0.41015625,
+      "learning_rate": 9.870674385552012e-06,
+      "loss": 0.0167,
+      "step": 350
+    },
+    {
+      "epoch": 0.40948741979669817,
+      "grad_norm": 0.271484375,
+      "learning_rate": 9.865344011682038e-06,
+      "loss": 0.0138,
+      "step": 355
+    },
+    {
+      "epoch": 0.41525484824453895,
+      "grad_norm": 0.306640625,
+      "learning_rate": 9.859907492997415e-06,
+      "loss": 0.0135,
+      "step": 360
+    },
+    {
+      "epoch": 0.4210222766923798,
+      "grad_norm": 0.2236328125,
+      "learning_rate": 9.854364948103991e-06,
+      "loss": 0.0127,
+      "step": 365
+    },
+    {
+      "epoch": 0.4267897051402206,
+      "grad_norm": 0.396484375,
+      "learning_rate": 9.848716497920742e-06,
+      "loss": 0.0162,
+      "step": 370
+    },
+    {
+      "epoch": 0.4325571335880614,
+      "grad_norm": 0.33203125,
+      "learning_rate": 9.842962265677123e-06,
+      "loss": 0.0171,
+      "step": 375
+    },
+    {
+      "epoch": 0.43832456203590225,
+      "grad_norm": 0.232421875,
+      "learning_rate": 9.837102376910383e-06,
+      "loss": 0.0131,
+      "step": 380
+    },
+    {
+      "epoch": 0.44409199048374304,
+      "grad_norm": 0.2392578125,
+      "learning_rate": 9.831136959462835e-06,
+      "loss": 0.0134,
+      "step": 385
+    },
+    {
+      "epoch": 0.4498594189315839,
+      "grad_norm": 0.392578125,
+      "learning_rate": 9.825066143479052e-06,
+      "loss": 0.0123,
+      "step": 390
+    },
+    {
+      "epoch": 0.4556268473794247,
+      "grad_norm": 0.53125,
+      "learning_rate": 9.818890061403034e-06,
+      "loss": 0.0169,
+      "step": 395
+    },
+    {
+      "epoch": 0.4613942758272655,
+      "grad_norm": 0.2373046875,
+      "learning_rate": 9.812608847975327e-06,
+      "loss": 0.0136,
+      "step": 400
+    },
+    {
+      "epoch": 0.46716170427510634,
+      "grad_norm": 0.275390625,
+      "learning_rate": 9.80622264023007e-06,
+      "loss": 0.0134,
+      "step": 405
+    },
+    {
+      "epoch": 0.4729291327229472,
+      "grad_norm": 0.240234375,
+      "learning_rate": 9.79973157749201e-06,
+      "loss": 0.0123,
+      "step": 410
+    },
+    {
+      "epoch": 0.47869656117078796,
+      "grad_norm": 0.29296875,
+      "learning_rate": 9.793135801373472e-06,
+      "loss": 0.0125,
+      "step": 415
+    },
+    {
+      "epoch": 0.4844639896186288,
+      "grad_norm": 0.361328125,
+      "learning_rate": 9.786435455771252e-06,
+      "loss": 0.0153,
+      "step": 420
+    },
+    {
+      "epoch": 0.49023141806646964,
+      "grad_norm": 0.5546875,
+      "learning_rate": 9.779630686863493e-06,
+      "loss": 0.0133,
+      "step": 425
+    },
+    {
+      "epoch": 0.4959988465143104,
+      "grad_norm": 0.314453125,
+      "learning_rate": 9.772721643106483e-06,
+      "loss": 0.0142,
+      "step": 430
+    },
+    {
+      "epoch": 0.5017662749621512,
+      "grad_norm": 0.26953125,
+      "learning_rate": 9.765708475231436e-06,
+      "loss": 0.0101,
+      "step": 435
+    },
+    {
+      "epoch": 0.507533703409992,
+      "grad_norm": 0.388671875,
+      "learning_rate": 9.758591336241176e-06,
+      "loss": 0.0255,
+      "step": 440
+    },
+    {
+      "epoch": 0.5133011318578329,
+      "grad_norm": 0.2890625,
+      "learning_rate": 9.75137038140682e-06,
+      "loss": 0.0109,
+      "step": 445
+    },
+    {
+      "epoch": 0.5190685603056737,
+      "grad_norm": 0.1953125,
+      "learning_rate": 9.744045768264387e-06,
+      "loss": 0.0119,
+      "step": 450
+    },
+    {
+      "epoch": 0.5248359887535146,
+      "grad_norm": 0.3125,
+      "learning_rate": 9.736617656611354e-06,
+      "loss": 0.011,
+      "step": 455
+    },
+    {
+      "epoch": 0.5306034172013554,
+      "grad_norm": 0.1982421875,
+      "learning_rate": 9.729086208503174e-06,
+      "loss": 0.0143,
+      "step": 460
+    },
+    {
+      "epoch": 0.5363708456491961,
+      "grad_norm": 0.2275390625,
+      "learning_rate": 9.721451588249744e-06,
+      "loss": 0.0105,
+      "step": 465
+    },
+    {
+      "epoch": 0.542138274097037,
+      "grad_norm": 0.25,
+      "learning_rate": 9.713713962411815e-06,
+      "loss": 0.0165,
+      "step": 470
+    },
+    {
+      "epoch": 0.5479057025448778,
+      "grad_norm": 0.2353515625,
+      "learning_rate": 9.705873499797358e-06,
+      "loss": 0.011,
+      "step": 475
+    },
+    {
+      "epoch": 0.5536731309927186,
+      "grad_norm": 0.23046875,
+      "learning_rate": 9.697930371457886e-06,
+      "loss": 0.011,
+      "step": 480
+    },
+    {
+      "epoch": 0.5594405594405595,
+      "grad_norm": 0.2060546875,
+      "learning_rate": 9.689884750684721e-06,
+      "loss": 0.012,
+      "step": 485
+    },
+    {
+      "epoch": 0.5652079878884002,
+      "grad_norm": 0.349609375,
+      "learning_rate": 9.681736813005207e-06,
+      "loss": 0.0101,
+      "step": 490
+    },
+    {
+      "epoch": 0.570975416336241,
+      "grad_norm": 0.275390625,
+      "learning_rate": 9.673486736178889e-06,
+      "loss": 0.0114,
+      "step": 495
+    },
+    {
+      "epoch": 0.5767428447840819,
+      "grad_norm": 0.193359375,
+      "learning_rate": 9.665134700193633e-06,
+      "loss": 0.0124,
+      "step": 500
+    },
+    {
+      "epoch": 0.5825102732319227,
+      "grad_norm": 0.1904296875,
+      "learning_rate": 9.656680887261693e-06,
+      "loss": 0.0124,
+      "step": 505
+    },
+    {
+      "epoch": 0.5882777016797636,
+      "grad_norm": 0.2265625,
+      "learning_rate": 9.648125481815744e-06,
+      "loss": 0.0151,
+      "step": 510
+    },
+    {
+      "epoch": 0.5940451301276044,
+      "grad_norm": 0.28515625,
+      "learning_rate": 9.639468670504854e-06,
+      "loss": 0.0209,
+      "step": 515
+    },
+    {
+      "epoch": 0.5998125585754451,
+      "grad_norm": 0.171875,
+      "learning_rate": 9.630710642190412e-06,
+      "loss": 0.0107,
+      "step": 520
+    },
+    {
+      "epoch": 0.605579987023286,
+      "grad_norm": 0.1689453125,
+      "learning_rate": 9.621851587942012e-06,
+      "loss": 0.0102,
+      "step": 525
+    },
+    {
+      "epoch": 0.6113474154711268,
+      "grad_norm": 0.224609375,
+      "learning_rate": 9.612891701033278e-06,
+      "loss": 0.013,
+      "step": 530
+    },
+    {
+      "epoch": 0.6171148439189676,
+      "grad_norm": 0.275390625,
+      "learning_rate": 9.603831176937645e-06,
+      "loss": 0.0104,
+      "step": 535
+    },
+    {
+      "epoch": 0.6228822723668085,
+      "grad_norm": 0.197265625,
+      "learning_rate": 9.594670213324114e-06,
+      "loss": 0.0121,
+      "step": 540
+    },
+    {
+      "epoch": 0.6286497008146492,
+      "grad_norm": 0.2021484375,
+      "learning_rate": 9.585409010052913e-06,
+      "loss": 0.012,
+      "step": 545
+    },
+    {
+      "epoch": 0.63441712926249,
+      "grad_norm": 0.255859375,
+      "learning_rate": 9.576047769171154e-06,
+      "loss": 0.0105,
+      "step": 550
+    },
+    {
+      "epoch": 0.6401845577103309,
+      "grad_norm": 0.19140625,
+      "learning_rate": 9.566586694908421e-06,
+      "loss": 0.0106,
+      "step": 555
+    },
+    {
+      "epoch": 0.6459519861581717,
+      "grad_norm": 0.275390625,
+      "learning_rate": 9.557025993672313e-06,
+      "loss": 0.0116,
+      "step": 560
+    },
+    {
+      "epoch": 0.6517194146060126,
+      "grad_norm": 0.2431640625,
+      "learning_rate": 9.547365874043939e-06,
+      "loss": 0.0127,
+      "step": 565
+    },
+    {
+      "epoch": 0.6574868430538534,
+      "grad_norm": 0.1708984375,
+      "learning_rate": 9.537606546773374e-06,
+      "loss": 0.0109,
+      "step": 570
+    },
+    {
+      "epoch": 0.6632542715016941,
+      "grad_norm": 0.30078125,
+      "learning_rate": 9.527748224775054e-06,
+      "loss": 0.0114,
+      "step": 575
+    },
+    {
+      "epoch": 0.669021699949535,
+      "grad_norm": 0.26171875,
+      "learning_rate": 9.517791123123141e-06,
+      "loss": 0.011,
+      "step": 580
+    },
+    {
+      "epoch": 0.6747891283973758,
+      "grad_norm": 0.2041015625,
+      "learning_rate": 9.507735459046812e-06,
+      "loss": 0.0125,
+      "step": 585
+    },
+    {
+      "epoch": 0.6805565568452167,
+      "grad_norm": 0.234375,
+      "learning_rate": 9.497581451925544e-06,
+      "loss": 0.0098,
+      "step": 590
+    },
+    {
+      "epoch": 0.6863239852930575,
+      "grad_norm": 0.2236328125,
+      "learning_rate": 9.487329323284306e-06,
+      "loss": 0.0118,
+      "step": 595
+    },
+    {
+      "epoch": 0.6920914137408983,
+      "grad_norm": 0.2421875,
+      "learning_rate": 9.476979296788746e-06,
+      "loss": 0.0099,
+      "step": 600
+    },
+    {
+      "epoch": 0.6978588421887391,
+      "grad_norm": 0.1982421875,
+      "learning_rate": 9.466531598240292e-06,
+      "loss": 0.014,
+      "step": 605
+    },
+    {
+      "epoch": 0.7036262706365799,
+      "grad_norm": 0.1904296875,
+      "learning_rate": 9.45598645557124e-06,
+      "loss": 0.0104,
+      "step": 610
+    },
+    {
+      "epoch": 0.7093936990844207,
+      "grad_norm": 0.2236328125,
+      "learning_rate": 9.445344098839774e-06,
+      "loss": 0.0106,
+      "step": 615
+    },
+    {
+      "epoch": 0.7151611275322616,
+      "grad_norm": 0.2255859375,
+      "learning_rate": 9.434604760224952e-06,
+      "loss": 0.0116,
+      "step": 620
+    },
+    {
+      "epoch": 0.7209285559801024,
+      "grad_norm": 0.23828125,
+      "learning_rate": 9.423768674021638e-06,
+      "loss": 0.0101,
+      "step": 625
+    },
+    {
+      "epoch": 0.7266959844279431,
+      "grad_norm": 0.1884765625,
+      "learning_rate": 9.412836076635385e-06,
+      "loss": 0.0104,
+      "step": 630
+    },
+    {
+      "epoch": 0.732463412875784,
+      "grad_norm": 0.1982421875,
+      "learning_rate": 9.401807206577292e-06,
+      "loss": 0.0115,
+      "step": 635
+    },
+    {
+      "epoch": 0.7382308413236248,
+      "grad_norm": 0.2265625,
+      "learning_rate": 9.390682304458782e-06,
+      "loss": 0.0098,
+      "step": 640
+    },
+    {
+      "epoch": 0.7439982697714657,
+      "grad_norm": 0.2216796875,
+      "learning_rate": 9.379461612986369e-06,
+      "loss": 0.0118,
+      "step": 645
+    },
+    {
+      "epoch": 0.7497656982193065,
+      "grad_norm": 0.169921875,
+      "learning_rate": 9.368145376956354e-06,
+      "loss": 0.0124,
+      "step": 650
+    },
+    {
+      "epoch": 0.7555331266671473,
+      "grad_norm": 0.1953125,
+      "learning_rate": 9.356733843249487e-06,
+      "loss": 0.0117,
+      "step": 655
+    },
+    {
+      "epoch": 0.7613005551149881,
+      "grad_norm": 0.283203125,
+      "learning_rate": 9.345227260825579e-06,
+      "loss": 0.0104,
+      "step": 660
+    },
+    {
+      "epoch": 0.7670679835628289,
+      "grad_norm": 0.20703125,
+      "learning_rate": 9.333625880718072e-06,
+      "loss": 0.0097,
+      "step": 665
+    },
+    {
+      "epoch": 0.7728354120106697,
+      "grad_norm": 0.1572265625,
+      "learning_rate": 9.321929956028565e-06,
+      "loss": 0.01,
+      "step": 670
+    },
+    {
+      "epoch": 0.7786028404585106,
+      "grad_norm": 0.2080078125,
+      "learning_rate": 9.310139741921287e-06,
+      "loss": 0.0163,
+      "step": 675
+    },
+    {
+      "epoch": 0.7843702689063514,
+      "grad_norm": 0.189453125,
+      "learning_rate": 9.298255495617535e-06,
+      "loss": 0.0102,
+      "step": 680
+    },
+    {
+      "epoch": 0.7901376973541921,
+      "grad_norm": 0.2001953125,
+      "learning_rate": 9.286277476390056e-06,
+      "loss": 0.0091,
+      "step": 685
+    },
+    {
+      "epoch": 0.795905125802033,
+      "grad_norm": 0.1689453125,
+      "learning_rate": 9.274205945557403e-06,
+      "loss": 0.0112,
+      "step": 690
+    },
+    {
+      "epoch": 0.8016725542498738,
+      "grad_norm": 0.330078125,
+      "learning_rate": 9.262041166478215e-06,
+      "loss": 0.0138,
+      "step": 695
+    },
+    {
+      "epoch": 0.8074399826977147,
+      "grad_norm": 0.162109375,
+      "learning_rate": 9.249783404545488e-06,
+      "loss": 0.0083,
+      "step": 700
+    },
+    {
+      "epoch": 0.8132074111455555,
+      "grad_norm": 0.2578125,
+      "learning_rate": 9.23743292718078e-06,
+      "loss": 0.009,
+      "step": 705
+    },
+    {
+      "epoch": 0.8189748395933963,
+      "grad_norm": 0.294921875,
+      "learning_rate": 9.224990003828377e-06,
+      "loss": 0.0119,
+      "step": 710
+    },
+    {
+      "epoch": 0.8247422680412371,
+      "grad_norm": 0.294921875,
+      "learning_rate": 9.212454905949406e-06,
+      "loss": 0.0106,
+      "step": 715
+    },
+    {
+      "epoch": 0.8305096964890779,
+      "grad_norm": 0.1767578125,
+      "learning_rate": 9.199827907015928e-06,
+      "loss": 0.0095,
+      "step": 720
+    },
+    {
+      "epoch": 0.8362771249369187,
+      "grad_norm": 0.1943359375,
+      "learning_rate": 9.187109282504961e-06,
+      "loss": 0.0094,
+      "step": 725
+    },
+    {
+      "epoch": 0.8420445533847596,
+      "grad_norm": 0.2177734375,
+      "learning_rate": 9.174299309892474e-06,
+      "loss": 0.0094,
+      "step": 730
+    },
+    {
+      "epoch": 0.8478119818326004,
+      "grad_norm": 0.2431640625,
+      "learning_rate": 9.161398268647329e-06,
+      "loss": 0.0139,
+      "step": 735
+    },
+    {
+      "epoch": 0.8535794102804412,
+      "grad_norm": 0.2265625,
+      "learning_rate": 9.148406440225193e-06,
+      "loss": 0.0113,
+      "step": 740
+    },
+    {
+      "epoch": 0.859346838728282,
+      "grad_norm": 0.1875,
+      "learning_rate": 9.135324108062391e-06,
+      "loss": 0.0095,
+      "step": 745
+    },
+    {
+      "epoch": 0.8651142671761228,
+      "grad_norm": 0.2041015625,
+      "learning_rate": 9.122151557569719e-06,
+      "loss": 0.009,
+      "step": 750
+    },
+    {
+      "epoch": 0.8708816956239637,
+      "grad_norm": 0.1796875,
+      "learning_rate": 9.108889076126226e-06,
+      "loss": 0.0103,
+      "step": 755
+    },
+    {
+      "epoch": 0.8766491240718045,
+      "grad_norm": 0.28515625,
+      "learning_rate": 9.09553695307294e-06,
+      "loss": 0.0104,
+      "step": 760
+    },
+    {
+      "epoch": 0.8824165525196453,
+      "grad_norm": 0.185546875,
+      "learning_rate": 9.082095479706556e-06,
+      "loss": 0.0084,
+      "step": 765
+    },
+    {
+      "epoch": 0.8881839809674861,
+      "grad_norm": 0.2041015625,
+      "learning_rate": 9.06856494927308e-06,
+      "loss": 0.0096,
+      "step": 770
+    },
+    {
+      "epoch": 0.8939514094153269,
+      "grad_norm": 0.185546875,
+      "learning_rate": 9.054945656961429e-06,
+      "loss": 0.0089,
+      "step": 775
+    },
+    {
+      "epoch": 0.8997188378631678,
+      "grad_norm": 0.15625,
+      "learning_rate": 9.041237899896999e-06,
+      "loss": 0.0087,
+      "step": 780
+    },
+    {
+      "epoch": 0.9054862663110086,
+      "grad_norm": 0.20703125,
+      "learning_rate": 9.027441977135178e-06,
+      "loss": 0.0115,
+      "step": 785
+    },
+    {
+      "epoch": 0.9112536947588494,
+      "grad_norm": 0.271484375,
+      "learning_rate": 9.013558189654819e-06,
+      "loss": 0.009,
+      "step": 790
+    },
+    {
+      "epoch": 0.9170211232066903,
+      "grad_norm": 0.216796875,
+      "learning_rate": 8.999586840351676e-06,
+      "loss": 0.0101,
+      "step": 795
+    },
+    {
+      "epoch": 0.922788551654531,
+      "grad_norm": 0.2041015625,
+      "learning_rate": 8.985528234031799e-06,
+      "loss": 0.0163,
+      "step": 800
+    },
+    {
+      "epoch": 0.9285559801023718,
+      "grad_norm": 0.25,
+      "learning_rate": 8.971382677404878e-06,
+      "loss": 0.0094,
+      "step": 805
+    },
+    {
+      "epoch": 0.9343234085502127,
+      "grad_norm": 0.166015625,
+      "learning_rate": 8.957150479077559e-06,
+      "loss": 0.0074,
+      "step": 810
+    },
+    {
+      "epoch": 0.9400908369980535,
+      "grad_norm": 0.189453125,
+      "learning_rate": 8.942831949546702e-06,
+      "loss": 0.0118,
+      "step": 815
+    },
+    {
+      "epoch": 0.9458582654458944,
+      "grad_norm": 0.2041015625,
+      "learning_rate": 8.928427401192618e-06,
+      "loss": 0.0084,
+      "step": 820
+    },
+    {
+      "epoch": 0.9516256938937351,
+      "grad_norm": 0.203125,
+      "learning_rate": 8.913937148272247e-06,
+      "loss": 0.0082,
+      "step": 825
+    },
+    {
+      "epoch": 0.9573931223415759,
+      "grad_norm": 0.259765625,
+      "learning_rate": 8.899361506912298e-06,
+      "loss": 0.0089,
+      "step": 830
+    },
+    {
+      "epoch": 0.9631605507894168,
+      "grad_norm": 0.1865234375,
+      "learning_rate": 8.884700795102365e-06,
+      "loss": 0.0114,
+      "step": 835
+    },
+    {
+      "epoch": 0.9689279792372576,
+      "grad_norm": 0.208984375,
+      "learning_rate": 8.869955332687975e-06,
+      "loss": 0.0084,
+      "step": 840
+    },
+    {
+      "epoch": 0.9746954076850984,
+      "grad_norm": 0.2197265625,
+      "learning_rate": 8.855125441363628e-06,
+      "loss": 0.0104,
+      "step": 845
+    },
+    {
+      "epoch": 0.9804628361329393,
+      "grad_norm": 0.203125,
+      "learning_rate": 8.840211444665754e-06,
+      "loss": 0.0094,
+      "step": 850
+    },
+    {
+      "epoch": 0.98623026458078,
+      "grad_norm": 0.1884765625,
+      "learning_rate": 8.825213667965678e-06,
+      "loss": 0.0094,
+      "step": 855
+    },
+    {
+      "epoch": 0.9919976930286208,
+      "grad_norm": 0.25390625,
+      "learning_rate": 8.810132438462509e-06,
+      "loss": 0.01,
+      "step": 860
+    },
+    {
+      "epoch": 0.9977651214764617,
+      "grad_norm": 0.263671875,
+      "learning_rate": 8.794968085176006e-06,
+      "loss": 0.0106,
+      "step": 865
+    },
+    {
+      "epoch": 1.0034604570687045,
+      "grad_norm": 0.22265625,
+      "learning_rate": 8.779720938939396e-06,
+      "loss": 0.0124,
+      "step": 870
+    },
+    {
+      "epoch": 1.0092278855165453,
+      "grad_norm": 0.1875,
+      "learning_rate": 8.764391332392163e-06,
+      "loss": 0.0088,
+      "step": 875
+    },
+    {
+      "epoch": 1.0149953139643861,
+      "grad_norm": 0.2294921875,
+      "learning_rate": 8.748979599972787e-06,
+      "loss": 0.0085,
+      "step": 880
+    },
+    {
+      "epoch": 1.020762742412227,
+      "grad_norm": 0.1923828125,
+      "learning_rate": 8.73348607791144e-06,
+      "loss": 0.0086,
+      "step": 885
+    },
+    {
+      "epoch": 1.0265301708600678,
+      "grad_norm": 0.23046875,
+      "learning_rate": 8.717911104222668e-06,
+      "loss": 0.0094,
+      "step": 890
+    },
+    {
+      "epoch": 1.0322975993079087,
+      "grad_norm": 0.1767578125,
+      "learning_rate": 8.702255018698e-06,
+      "loss": 0.0085,
+      "step": 895
+    },
+    {
+      "epoch": 1.0380650277557495,
+      "grad_norm": 0.2080078125,
+      "learning_rate": 8.686518162898547e-06,
+      "loss": 0.0087,
+      "step": 900
+    },
+    {
+      "epoch": 1.0438324562035903,
+      "grad_norm": 0.224609375,
+      "learning_rate": 8.670700880147544e-06,
+      "loss": 0.0088,
+      "step": 905
+    },
+    {
+      "epoch": 1.049599884651431,
+      "grad_norm": 0.2041015625,
+      "learning_rate": 8.65480351552286e-06,
+      "loss": 0.0085,
+      "step": 910
+    },
+    {
+      "epoch": 1.0553673130992718,
+      "grad_norm": 0.1767578125,
+      "learning_rate": 8.638826415849469e-06,
+      "loss": 0.0088,
+      "step": 915
+    },
+    {
+      "epoch": 1.0611347415471126,
+      "grad_norm": 0.158203125,
+      "learning_rate": 8.622769929691889e-06,
+      "loss": 0.011,
+      "step": 920
+    },
+    {
+      "epoch": 1.0669021699949535,
+      "grad_norm": 0.181640625,
+      "learning_rate": 8.606634407346575e-06,
+      "loss": 0.009,
+      "step": 925
+    },
+    {
+      "epoch": 1.0726695984427943,
+      "grad_norm": 0.1806640625,
+      "learning_rate": 8.590420200834271e-06,
+      "loss": 0.0086,
+      "step": 930
+    },
+    {
+      "epoch": 1.0784370268906351,
+      "grad_norm": 0.267578125,
+      "learning_rate": 8.574127663892334e-06,
+      "loss": 0.0093,
+      "step": 935
+    },
+    {
+      "epoch": 1.084204455338476,
+      "grad_norm": 0.228515625,
+      "learning_rate": 8.557757151967025e-06,
+      "loss": 0.0083,
+      "step": 940
+    },
+    {
+      "epoch": 1.0899718837863168,
+      "grad_norm": 0.1904296875,
+      "learning_rate": 8.541309022205738e-06,
+      "loss": 0.0094,
+      "step": 945
+    },
+    {
+      "epoch": 1.0957393122341577,
+      "grad_norm": 0.1806640625,
+      "learning_rate": 8.524783633449224e-06,
+      "loss": 0.0073,
+      "step": 950
+    },
+    {
+      "epoch": 1.1015067406819985,
+      "grad_norm": 0.158203125,
+      "learning_rate": 8.508181346223749e-06,
+      "loss": 0.0076,
+      "step": 955
+    },
+    {
+      "epoch": 1.1072741691298393,
+      "grad_norm": 0.173828125,
+      "learning_rate": 8.491502522733243e-06,
+      "loss": 0.0081,
+      "step": 960
+    },
+    {
+      "epoch": 1.11304159757768,
+      "grad_norm": 0.1845703125,
+      "learning_rate": 8.474747526851383e-06,
+      "loss": 0.007,
+      "step": 965
+    },
+    {
+      "epoch": 1.1188090260255208,
+      "grad_norm": 0.2109375,
+      "learning_rate": 8.457916724113667e-06,
+      "loss": 0.0087,
+      "step": 970
+    },
+    {
+      "epoch": 1.1245764544733616,
+      "grad_norm": 0.1513671875,
+      "learning_rate": 8.441010481709424e-06,
+      "loss": 0.0073,
+      "step": 975
+    },
+    {
+      "epoch": 1.1303438829212025,
+      "grad_norm": 0.1884765625,
+      "learning_rate": 8.424029168473829e-06,
+      "loss": 0.009,
+      "step": 980
+    },
+    {
+      "epoch": 1.1361113113690433,
+      "grad_norm": 0.240234375,
+      "learning_rate": 8.406973154879826e-06,
+      "loss": 0.0085,
+      "step": 985
+    },
+    {
+      "epoch": 1.1418787398168841,
+      "grad_norm": 0.150390625,
+      "learning_rate": 8.389842813030065e-06,
+      "loss": 0.0076,
+      "step": 990
+    },
+    {
+      "epoch": 1.147646168264725,
+      "grad_norm": 0.1923828125,
+      "learning_rate": 8.372638516648782e-06,
+      "loss": 0.0094,
+      "step": 995
+    },
+    {
+      "epoch": 1.1534135967125658,
+      "grad_norm": 0.2099609375,
+      "learning_rate": 8.355360641073637e-06,
+      "loss": 0.0086,
+      "step": 1000
+    },
+    {
+      "epoch": 1.1591810251604067,
+      "grad_norm": 0.14453125,
+      "learning_rate": 8.338009563247535e-06,
+      "loss": 0.0084,
+      "step": 1005
+    },
+    {
+      "epoch": 1.1649484536082475,
+      "grad_norm": 0.197265625,
+      "learning_rate": 8.3205856617104e-06,
+      "loss": 0.0111,
+      "step": 1010
+    },
+    {
+      "epoch": 1.1707158820560881,
+      "grad_norm": 0.2109375,
+      "learning_rate": 8.30308931659091e-06,
+      "loss": 0.0086,
+      "step": 1015
+    },
+    {
+      "epoch": 1.176483310503929,
+      "grad_norm": 0.1982421875,
+      "learning_rate": 8.285520909598213e-06,
+      "loss": 0.0078,
+      "step": 1020
+    },
+    {
+      "epoch": 1.1822507389517698,
+      "grad_norm": 0.203125,
+      "learning_rate": 8.267880824013591e-06,
+      "loss": 0.0082,
+      "step": 1025
+    },
+    {
+      "epoch": 1.1880181673996106,
+      "grad_norm": 0.201171875,
+      "learning_rate": 8.250169444682109e-06,
+      "loss": 0.0084,
+      "step": 1030
+    },
+    {
+      "epoch": 1.1937855958474515,
+      "grad_norm": 0.2158203125,
+      "learning_rate": 8.232387158004205e-06,
+      "loss": 0.0071,
+      "step": 1035
+    },
+    {
+      "epoch": 1.1995530242952923,
+      "grad_norm": 0.224609375,
+      "learning_rate": 8.214534351927275e-06,
+      "loss": 0.0096,
+      "step": 1040
+    },
+    {
+      "epoch": 1.2053204527431332,
+      "grad_norm": 0.1806640625,
+      "learning_rate": 8.196611415937196e-06,
+      "loss": 0.0083,
+      "step": 1045
+    },
+    {
+      "epoch": 1.211087881190974,
+      "grad_norm": 0.201171875,
+      "learning_rate": 8.178618741049841e-06,
+      "loss": 0.0101,
+      "step": 1050
+    },
+    {
+      "epoch": 1.2168553096388148,
+      "grad_norm": 0.25390625,
+      "learning_rate": 8.160556719802535e-06,
+      "loss": 0.0078,
+      "step": 1055
+    },
+    {
+      "epoch": 1.2226227380866557,
+      "grad_norm": 0.16015625,
+      "learning_rate": 8.142425746245503e-06,
+      "loss": 0.0078,
+      "step": 1060
+    },
+    {
+      "epoch": 1.2283901665344965,
+      "grad_norm": 0.1826171875,
+      "learning_rate": 8.124226215933268e-06,
+      "loss": 0.0101,
+      "step": 1065
+    },
+    {
+      "epoch": 1.2341575949823373,
+      "grad_norm": 0.2470703125,
+      "learning_rate": 8.10595852591602e-06,
+      "loss": 0.008,
+      "step": 1070
+    },
+    {
+      "epoch": 1.2399250234301782,
+      "grad_norm": 0.25,
+      "learning_rate": 8.08762307473096e-06,
+      "loss": 0.0075,
+      "step": 1075
+    },
+    {
+      "epoch": 1.2456924518780188,
+      "grad_norm": 0.2080078125,
+      "learning_rate": 8.069220262393597e-06,
+      "loss": 0.0105,
+      "step": 1080
+    },
+    {
+      "epoch": 1.2514598803258596,
+      "grad_norm": 0.2294921875,
+      "learning_rate": 8.050750490389026e-06,
+      "loss": 0.0076,
+      "step": 1085
+    },
+    {
+      "epoch": 1.2572273087737005,
+      "grad_norm": 0.173828125,
+      "learning_rate": 8.03221416166317e-06,
+      "loss": 0.0083,
+      "step": 1090
+    },
+    {
+      "epoch": 1.2629947372215413,
+      "grad_norm": 0.2080078125,
+      "learning_rate": 8.013611680613983e-06,
+      "loss": 0.0082,
+      "step": 1095
+    },
+    {
+      "epoch": 1.2687621656693822,
+      "grad_norm": 0.1953125,
+      "learning_rate": 7.994943453082636e-06,
+      "loss": 0.0097,
+      "step": 1100
+    },
+    {
+      "epoch": 1.274529594117223,
+      "grad_norm": 0.193359375,
+      "learning_rate": 7.976209886344654e-06,
+      "loss": 0.0087,
+      "step": 1105
+    },
+    {
+      "epoch": 1.2802970225650638,
+      "grad_norm": 0.197265625,
+      "learning_rate": 7.957411389101045e-06,
+      "loss": 0.0077,
+      "step": 1110
+    },
+    {
+      "epoch": 1.2860644510129047,
+      "grad_norm": 0.1953125,
+      "learning_rate": 7.938548371469362e-06,
+      "loss": 0.01,
+      "step": 1115
+    },
+    {
+      "epoch": 1.2918318794607455,
+      "grad_norm": 0.162109375,
+      "learning_rate": 7.919621244974773e-06,
+      "loss": 0.0189,
+      "step": 1120
+    },
+    {
+      "epoch": 1.2975993079085861,
+      "grad_norm": 0.296875,
+      "learning_rate": 7.900630422541079e-06,
+      "loss": 0.0106,
+      "step": 1125
+    },
+    {
+      "epoch": 1.303366736356427,
+      "grad_norm": 0.1728515625,
+      "learning_rate": 7.881576318481702e-06,
+      "loss": 0.0085,
+      "step": 1130
+    },
+    {
+      "epoch": 1.3091341648042678,
+      "grad_norm": 0.2177734375,
+      "learning_rate": 7.862459348490645e-06,
+      "loss": 0.0095,
+      "step": 1135
+    },
+    {
+      "epoch": 1.3149015932521086,
+      "grad_norm": 0.158203125,
+      "learning_rate": 7.843279929633434e-06,
+      "loss": 0.009,
+      "step": 1140
+    },
+    {
+      "epoch": 1.3206690216999495,
+      "grad_norm": 0.1767578125,
+      "learning_rate": 7.824038480338002e-06,
+      "loss": 0.0078,
+      "step": 1145
+    },
+    {
+      "epoch": 1.3264364501477903,
+      "grad_norm": 0.193359375,
+      "learning_rate": 7.804735420385578e-06,
+      "loss": 0.0091,
+      "step": 1150
+    },
+    {
+      "epoch": 1.3322038785956312,
+      "grad_norm": 0.2421875,
+      "learning_rate": 7.785371170901512e-06,
+      "loss": 0.0084,
+      "step": 1155
+    },
+    {
+      "epoch": 1.337971307043472,
+      "grad_norm": 0.181640625,
+      "learning_rate": 7.765946154346105e-06,
+      "loss": 0.012,
+      "step": 1160
+    },
+    {
+      "epoch": 1.3437387354913128,
+      "grad_norm": 0.1435546875,
+      "learning_rate": 7.746460794505375e-06,
+      "loss": 0.0082,
+      "step": 1165
+    },
+    {
+      "epoch": 1.3495061639391537,
+      "grad_norm": 0.203125,
+      "learning_rate": 7.726915516481824e-06,
+      "loss": 0.0079,
+      "step": 1170
+    },
+    {
+      "epoch": 1.3552735923869945,
+      "grad_norm": 0.1796875,
+      "learning_rate": 7.70731074668516e-06,
+      "loss": 0.0077,
+      "step": 1175
+    },
+    {
+      "epoch": 1.3610410208348354,
+      "grad_norm": 0.1494140625,
+      "learning_rate": 7.68764691282299e-06,
+      "loss": 0.0079,
+      "step": 1180
+    },
+    {
+      "epoch": 1.3668084492826762,
+      "grad_norm": 0.1455078125,
+      "learning_rate": 7.667924443891498e-06,
+      "loss": 0.0086,
+      "step": 1185
+    },
+    {
+      "epoch": 1.372575877730517,
+      "grad_norm": 0.193359375,
+      "learning_rate": 7.648143770166074e-06,
+      "loss": 0.0075,
+      "step": 1190
+    },
+    {
+      "epoch": 1.3783433061783577,
+      "grad_norm": 0.16796875,
+      "learning_rate": 7.628305323191942e-06,
+      "loss": 0.0085,
+      "step": 1195
+    },
+    {
+      "epoch": 1.3841107346261985,
+      "grad_norm": 0.2099609375,
+      "learning_rate": 7.608409535774725e-06,
+      "loss": 0.0081,
+      "step": 1200
+    },
+    {
+      "epoch": 1.3898781630740393,
+      "grad_norm": 0.11962890625,
+      "learning_rate": 7.588456841971026e-06,
+      "loss": 0.0078,
+      "step": 1205
+    },
+    {
+      "epoch": 1.3956455915218802,
+      "grad_norm": 0.1494140625,
+      "learning_rate": 7.568447677078937e-06,
+      "loss": 0.0085,
+      "step": 1210
+    },
+    {
+      "epoch": 1.401413019969721,
+      "grad_norm": 0.20703125,
+      "learning_rate": 7.54838247762856e-06,
+      "loss": 0.0074,
+      "step": 1215
+    },
+    {
+      "epoch": 1.4071804484175618,
+      "grad_norm": 0.2119140625,
+      "learning_rate": 7.528261681372468e-06,
+      "loss": 0.0072,
+      "step": 1220
+    },
+    {
+      "epoch": 1.4129478768654027,
+      "grad_norm": 0.1982421875,
+      "learning_rate": 7.508085727276169e-06,
+      "loss": 0.0095,
+      "step": 1225
+    },
+    {
+      "epoch": 1.4187153053132435,
+      "grad_norm": 0.2294921875,
+      "learning_rate": 7.487855055508516e-06,
+      "loss": 0.0083,
+      "step": 1230
+    },
+    {
+      "epoch": 1.4244827337610841,
+      "grad_norm": 0.125,
+      "learning_rate": 7.467570107432115e-06,
+      "loss": 0.0073,
+      "step": 1235
+    },
+    {
+      "epoch": 1.430250162208925,
+      "grad_norm": 0.2001953125,
+      "learning_rate": 7.447231325593689e-06,
+      "loss": 0.0085,
+      "step": 1240
+    },
+    {
+      "epoch": 1.4360175906567658,
+      "grad_norm": 0.1708984375,
+      "learning_rate": 7.426839153714429e-06,
+      "loss": 0.0084,
+      "step": 1245
+    },
+    {
+      "epoch": 1.4417850191046067,
+      "grad_norm": 0.19921875,
+      "learning_rate": 7.406394036680306e-06,
+      "loss": 0.0071,
+      "step": 1250
+    },
+    {
+      "epoch": 1.4475524475524475,
+      "grad_norm": 0.2294921875,
+      "learning_rate": 7.385896420532372e-06,
+      "loss": 0.0086,
+      "step": 1255
+    },
+    {
+      "epoch": 1.4533198760002883,
+      "grad_norm": 0.2392578125,
+      "learning_rate": 7.365346752457026e-06,
+      "loss": 0.0095,
+      "step": 1260
+    },
+    {
+      "epoch": 1.4590873044481292,
+      "grad_norm": 0.1494140625,
+      "learning_rate": 7.3447454807762565e-06,
+      "loss": 0.0073,
+      "step": 1265
+    },
+    {
+      "epoch": 1.46485473289597,
+      "grad_norm": 0.1787109375,
+      "learning_rate": 7.324093054937864e-06,
+      "loss": 0.0088,
+      "step": 1270
+    },
+    {
+      "epoch": 1.4706221613438109,
+      "grad_norm": 0.2060546875,
+      "learning_rate": 7.3033899255056575e-06,
+      "loss": 0.0093,
+      "step": 1275
+    },
+    {
+      "epoch": 1.4763895897916517,
+      "grad_norm": 0.2353515625,
+      "learning_rate": 7.282636544149613e-06,
+      "loss": 0.0094,
+      "step": 1280
+    },
+    {
+      "epoch": 1.4821570182394925,
+      "grad_norm": 0.1982421875,
+      "learning_rate": 7.261833363636036e-06,
+      "loss": 0.0083,
+      "step": 1285
+    },
+    {
+      "epoch": 1.4879244466873334,
+      "grad_norm": 0.2578125,
+      "learning_rate": 7.24098083781767e-06,
+      "loss": 0.01,
+      "step": 1290
+    },
+    {
+      "epoch": 1.4936918751351742,
+      "grad_norm": 0.154296875,
+      "learning_rate": 7.220079421623805e-06,
+      "loss": 0.0074,
+      "step": 1295
+    },
+    {
+      "epoch": 1.499459303583015,
+      "grad_norm": 0.189453125,
+      "learning_rate": 7.199129571050345e-06,
+      "loss": 0.0115,
+      "step": 1300
+    },
+    {
+      "epoch": 1.5052267320308559,
+      "grad_norm": 0.1611328125,
+      "learning_rate": 7.178131743149868e-06,
+      "loss": 0.0076,
+      "step": 1305
+    },
+    {
+      "epoch": 1.5109941604786967,
+      "grad_norm": 0.1650390625,
+      "learning_rate": 7.157086396021641e-06,
+      "loss": 0.0083,
+      "step": 1310
+    },
+    {
+      "epoch": 1.5167615889265373,
+      "grad_norm": 0.267578125,
+      "learning_rate": 7.135993988801644e-06,
+      "loss": 0.0084,
+      "step": 1315
+    },
+    {
+      "epoch": 1.5225290173743782,
+      "grad_norm": 0.1943359375,
+      "learning_rate": 7.114854981652537e-06,
+      "loss": 0.0072,
+      "step": 1320
+    },
+    {
+      "epoch": 1.528296445822219,
+      "grad_norm": 0.275390625,
+      "learning_rate": 7.093669835753632e-06,
+      "loss": 0.007,
+      "step": 1325
+    },
+    {
+      "epoch": 1.5340638742700599,
+      "grad_norm": 0.2451171875,
+      "learning_rate": 7.072439013290824e-06,
+      "loss": 0.0084,
+      "step": 1330
+    },
+    {
+      "epoch": 1.5398313027179007,
+      "grad_norm": 0.2353515625,
+      "learning_rate": 7.051162977446513e-06,
+      "loss": 0.0074,
+      "step": 1335
+    },
+    {
+      "epoch": 1.5455987311657413,
+      "grad_norm": 0.11376953125,
+      "learning_rate": 7.029842192389498e-06,
+      "loss": 0.0097,
+      "step": 1340
+    },
+    {
+      "epoch": 1.5513661596135822,
+      "grad_norm": 0.2421875,
+      "learning_rate": 7.008477123264849e-06,
+      "loss": 0.0086,
+      "step": 1345
+    },
+    {
+      "epoch": 1.557133588061423,
+      "grad_norm": 0.201171875,
+      "learning_rate": 6.987068236183756e-06,
+      "loss": 0.0096,
+      "step": 1350
+    },
+    {
+      "epoch": 1.5629010165092638,
+      "grad_norm": 0.29296875,
+      "learning_rate": 6.965615998213371e-06,
+      "loss": 0.0083,
+      "step": 1355
+    },
+    {
+      "epoch": 1.5686684449571047,
+      "grad_norm": 0.205078125,
+      "learning_rate": 6.944120877366605e-06,
+      "loss": 0.0081,
+      "step": 1360
+    },
+    {
+      "epoch": 1.5744358734049455,
+      "grad_norm": 0.1513671875,
+      "learning_rate": 6.922583342591924e-06,
+      "loss": 0.0083,
+      "step": 1365
+    },
+    {
+      "epoch": 1.5802033018527863,
+      "grad_norm": 0.2109375,
+      "learning_rate": 6.901003863763121e-06,
+      "loss": 0.0076,
+      "step": 1370
+    },
+    {
+      "epoch": 1.5859707303006272,
+      "grad_norm": 0.1865234375,
+      "learning_rate": 6.87938291166906e-06,
+      "loss": 0.0072,
+      "step": 1375
+    },
+    {
+      "epoch": 1.591738158748468,
+      "grad_norm": 0.142578125,
+      "learning_rate": 6.857720958003406e-06,
+      "loss": 0.0073,
+      "step": 1380
+    },
+    {
+      "epoch": 1.5975055871963089,
+      "grad_norm": 0.2470703125,
+      "learning_rate": 6.836018475354339e-06,
+      "loss": 0.0114,
+      "step": 1385
+    },
+    {
+      "epoch": 1.6032730156441497,
+      "grad_norm": 0.1845703125,
+      "learning_rate": 6.814275937194233e-06,
+      "loss": 0.007,
+      "step": 1390
+    },
+    {
+      "epoch": 1.6090404440919905,
+      "grad_norm": 0.1474609375,
+      "learning_rate": 6.792493817869339e-06,
+      "loss": 0.007,
+      "step": 1395
+    },
+    {
+      "epoch": 1.6148078725398314,
+      "grad_norm": 0.197265625,
+      "learning_rate": 6.7706725925894264e-06,
+      "loss": 0.0083,
+      "step": 1400
+    },
+    {
+      "epoch": 1.6205753009876722,
+      "grad_norm": 0.1962890625,
+      "learning_rate": 6.748812737417428e-06,
+      "loss": 0.0093,
+      "step": 1405
+    },
+    {
+      "epoch": 1.626342729435513,
+      "grad_norm": 0.2060546875,
+      "learning_rate": 6.726914729259034e-06,
+      "loss": 0.0107,
+      "step": 1410
+    },
+    {
+      "epoch": 1.632110157883354,
+      "grad_norm": 0.2021484375,
+      "learning_rate": 6.704979045852311e-06,
+      "loss": 0.0062,
+      "step": 1415
+    },
+    {
+      "epoch": 1.6378775863311947,
+      "grad_norm": 0.1650390625,
+      "learning_rate": 6.683006165757262e-06,
+      "loss": 0.0075,
+      "step": 1420
+    },
+    {
+      "epoch": 1.6436450147790354,
+      "grad_norm": 0.1640625,
+      "learning_rate": 6.660996568345392e-06,
+      "loss": 0.0105,
+      "step": 1425
+    },
+    {
+      "epoch": 1.6494124432268762,
+      "grad_norm": 0.1396484375,
+      "learning_rate": 6.63895073378925e-06,
+      "loss": 0.0061,
+      "step": 1430
+    },
+    {
+      "epoch": 1.655179871674717,
+      "grad_norm": 0.1728515625,
+      "learning_rate": 6.6168691430519524e-06,
+      "loss": 0.0084,
+      "step": 1435
+    },
+    {
+      "epoch": 1.6609473001225579,
+      "grad_norm": 0.142578125,
+      "learning_rate": 6.594752277876695e-06,
+      "loss": 0.0074,
+      "step": 1440
+    },
+    {
+      "epoch": 1.6667147285703987,
+      "grad_norm": 0.177734375,
+      "learning_rate": 6.572600620776232e-06,
+      "loss": 0.0072,
+      "step": 1445
+    },
+    {
+      "epoch": 1.6724821570182395,
+      "grad_norm": 0.1328125,
+      "learning_rate": 6.55041465502236e-06,
+      "loss": 0.0064,
+      "step": 1450
+    },
+    {
+      "epoch": 1.6782495854660802,
+      "grad_norm": 0.2021484375,
+      "learning_rate": 6.52819486463537e-06,
+      "loss": 0.0077,
+      "step": 1455
+    },
+    {
+      "epoch": 1.684017013913921,
+      "grad_norm": 0.18359375,
+      "learning_rate": 6.505941734373489e-06,
+      "loss": 0.0078,
+      "step": 1460
+    },
+    {
+      "epoch": 1.6897844423617618,
+      "grad_norm": 0.197265625,
+      "learning_rate": 6.4836557497222995e-06,
+      "loss": 0.007,
+      "step": 1465
+    },
+    {
+      "epoch": 1.6955518708096027,
+      "grad_norm": 0.181640625,
+      "learning_rate": 6.461337396884158e-06,
+      "loss": 0.0088,
+      "step": 1470
+    },
+    {
+      "epoch": 1.7013192992574435,
+      "grad_norm": 0.1923828125,
+      "learning_rate": 6.438987162767578e-06,
+      "loss": 0.0151,
+      "step": 1475
+    },
+    {
+      "epoch": 1.7070867277052844,
+      "grad_norm": 0.1591796875,
+      "learning_rate": 6.416605534976614e-06,
+      "loss": 0.007,
+      "step": 1480
+    },
+    {
+      "epoch": 1.7128541561531252,
+      "grad_norm": 0.125,
+      "learning_rate": 6.3941930018002175e-06,
+      "loss": 0.0075,
+      "step": 1485
+    },
+    {
+      "epoch": 1.718621584600966,
+      "grad_norm": 0.2353515625,
+      "learning_rate": 6.371750052201588e-06,
+      "loss": 0.0085,
+      "step": 1490
+    },
+    {
+      "epoch": 1.7243890130488069,
+      "grad_norm": 0.1435546875,
+      "learning_rate": 6.349277175807506e-06,
+      "loss": 0.0074,
+      "step": 1495
+    },
+    {
+      "epoch": 1.7301564414966477,
+      "grad_norm": 0.169921875,
+      "learning_rate": 6.326774862897653e-06,
+      "loss": 0.0069,
+      "step": 1500
+    },
+    {
+      "epoch": 1.7359238699444886,
+      "grad_norm": 0.1279296875,
+      "learning_rate": 6.304243604393908e-06,
+      "loss": 0.0072,
+      "step": 1505
+    },
+    {
+      "epoch": 1.7416912983923294,
+      "grad_norm": 0.1376953125,
+      "learning_rate": 6.281683891849645e-06,
+      "loss": 0.0072,
+      "step": 1510
+    },
+    {
+      "epoch": 1.7474587268401702,
+      "grad_norm": 0.232421875,
+      "learning_rate": 6.259096217439002e-06,
+      "loss": 0.0098,
+      "step": 1515
+    },
+    {
+      "epoch": 1.753226155288011,
+      "grad_norm": 0.16796875,
+      "learning_rate": 6.2364810739461525e-06,
+      "loss": 0.0156,
+      "step": 1520
+    },
+    {
+      "epoch": 1.758993583735852,
+      "grad_norm": 0.16796875,
+      "learning_rate": 6.213838954754543e-06,
+      "loss": 0.0072,
+      "step": 1525
+    },
+    {
+      "epoch": 1.7647610121836927,
+      "grad_norm": 0.189453125,
+      "learning_rate": 6.191170353836143e-06,
+      "loss": 0.0077,
+      "step": 1530
+    },
+    {
+      "epoch": 1.7705284406315334,
+      "grad_norm": 0.1376953125,
+      "learning_rate": 6.1684757657406545e-06,
+      "loss": 0.0082,
+      "step": 1535
+    },
+    {
+      "epoch": 1.7762958690793742,
+      "grad_norm": 0.1787109375,
+      "learning_rate": 6.145755685584731e-06,
+      "loss": 0.008,
+      "step": 1540
+    },
+    {
+      "epoch": 1.782063297527215,
+      "grad_norm": 0.1943359375,
+      "learning_rate": 6.1230106090411755e-06,
+      "loss": 0.0081,
+      "step": 1545
+    },
+    {
+      "epoch": 1.7878307259750559,
+      "grad_norm": 0.1572265625,
+      "learning_rate": 6.100241032328125e-06,
+      "loss": 0.0101,
+      "step": 1550
+    },
+    {
+      "epoch": 1.7935981544228967,
+      "grad_norm": 0.2578125,
+      "learning_rate": 6.077447452198219e-06,
+      "loss": 0.0063,
+      "step": 1555
+    },
+    {
+      "epoch": 1.7993655828707376,
+      "grad_norm": 0.173828125,
+      "learning_rate": 6.054630365927775e-06,
+      "loss": 0.0076,
+      "step": 1560
+    },
+    {
+      "epoch": 1.8051330113185782,
+      "grad_norm": 0.23046875,
+      "learning_rate": 6.0317902713059275e-06,
+      "loss": 0.0085,
+      "step": 1565
+    },
+    {
+      "epoch": 1.810900439766419,
+      "grad_norm": 0.1845703125,
+      "learning_rate": 6.008927666623775e-06,
+      "loss": 0.0078,
+      "step": 1570
+    },
+    {
+      "epoch": 1.8166678682142599,
+      "grad_norm": 0.1533203125,
+      "learning_rate": 5.986043050663507e-06,
+      "loss": 0.0075,
+      "step": 1575
+    },
+    {
+      "epoch": 1.8224352966621007,
+      "grad_norm": 0.19921875,
+      "learning_rate": 5.963136922687523e-06,
+      "loss": 0.0099,
+      "step": 1580
+    },
+    {
+      "epoch": 1.8282027251099415,
+      "grad_norm": 0.1904296875,
+      "learning_rate": 5.940209782427535e-06,
+      "loss": 0.0088,
+      "step": 1585
+    },
+    {
+      "epoch": 1.8339701535577824,
+      "grad_norm": 0.177734375,
+      "learning_rate": 5.9172621300736755e-06,
+      "loss": 0.0088,
+      "step": 1590
+    },
+    {
+      "epoch": 1.8397375820056232,
+      "grad_norm": 0.169921875,
+      "learning_rate": 5.894294466263576e-06,
+      "loss": 0.0081,
+      "step": 1595
+    },
+    {
+      "epoch": 1.845505010453464,
+      "grad_norm": 0.171875,
+      "learning_rate": 5.871307292071449e-06,
+      "loss": 0.0079,
+      "step": 1600
+    },
+    {
+      "epoch": 1.8512724389013049,
+      "grad_norm": 0.1328125,
+      "learning_rate": 5.848301108997154e-06,
+      "loss": 0.0081,
+      "step": 1605
+    },
+    {
+      "epoch": 1.8570398673491457,
+      "grad_norm": 0.1162109375,
+      "learning_rate": 5.8252764189552615e-06,
+      "loss": 0.0068,
+      "step": 1610
+    },
+    {
+      "epoch": 1.8628072957969866,
+      "grad_norm": 0.1796875,
+      "learning_rate": 5.802233724264094e-06,
+      "loss": 0.0083,
+      "step": 1615
+    },
+    {
+      "epoch": 1.8685747242448274,
+      "grad_norm": 0.1630859375,
+      "learning_rate": 5.779173527634778e-06,
+      "loss": 0.0078,
+      "step": 1620
+    },
+    {
+      "epoch": 1.8743421526926682,
+      "grad_norm": 0.24609375,
+      "learning_rate": 5.756096332160267e-06,
+      "loss": 0.0067,
+      "step": 1625
+    },
+    {
+      "epoch": 1.880109581140509,
+      "grad_norm": 0.1728515625,
+      "learning_rate": 5.7330026413043726e-06,
+      "loss": 0.0073,
+      "step": 1630
+    },
+    {
+      "epoch": 1.88587700958835,
+      "grad_norm": 0.1650390625,
+      "learning_rate": 5.709892958890777e-06,
+      "loss": 0.0076,
+      "step": 1635
+    },
+    {
+      "epoch": 1.8916444380361908,
+      "grad_norm": 0.1591796875,
+      "learning_rate": 5.686767789092041e-06,
+      "loss": 0.0075,
+      "step": 1640
+    },
+    {
+      "epoch": 1.8974118664840316,
+      "grad_norm": 0.2099609375,
+      "learning_rate": 5.663627636418611e-06,
+      "loss": 0.007,
+      "step": 1645
+    },
+    {
+      "epoch": 1.9031792949318722,
+      "grad_norm": 0.24609375,
+      "learning_rate": 5.640473005707801e-06,
+      "loss": 0.0087,
+      "step": 1650
+    },
+    {
+      "epoch": 1.908946723379713,
+      "grad_norm": 0.16796875,
+      "learning_rate": 5.617304402112788e-06,
+      "loss": 0.0081,
+      "step": 1655
+    },
+    {
+      "epoch": 1.914714151827554,
+      "grad_norm": 0.23046875,
+      "learning_rate": 5.594122331091591e-06,
+      "loss": 0.0098,
+      "step": 1660
+    },
+    {
+      "epoch": 1.9204815802753947,
+      "grad_norm": 0.1796875,
+      "learning_rate": 5.570927298396035e-06,
+      "loss": 0.0075,
+      "step": 1665
+    },
+    {
+      "epoch": 1.9262490087232356,
+      "grad_norm": 0.185546875,
+      "learning_rate": 5.54771981006073e-06,
+      "loss": 0.0077,
+      "step": 1670
+    },
+    {
+      "epoch": 1.9320164371710762,
+      "grad_norm": 0.134765625,
+      "learning_rate": 5.524500372392021e-06,
+      "loss": 0.0086,
+      "step": 1675
+    },
+    {
+      "epoch": 1.937783865618917,
+      "grad_norm": 0.1669921875,
+      "learning_rate": 5.501269491956946e-06,
+      "loss": 0.0099,
+      "step": 1680
+    },
+    {
+      "epoch": 1.9435512940667579,
+      "grad_norm": 0.17578125,
+      "learning_rate": 5.478027675572188e-06,
+      "loss": 0.0074,
+      "step": 1685
+    },
+    {
+      "epoch": 1.9493187225145987,
+      "grad_norm": 0.1552734375,
+      "learning_rate": 5.454775430293008e-06,
+      "loss": 0.0073,
+      "step": 1690
+    },
+    {
+      "epoch": 1.9550861509624395,
+      "grad_norm": 0.1796875,
+      "learning_rate": 5.431513263402192e-06,
+      "loss": 0.009,
+      "step": 1695
+    },
+    {
+      "epoch": 1.9608535794102804,
+      "grad_norm": 0.205078125,
+      "learning_rate": 5.4082416823989825e-06,
+      "loss": 0.0065,
+      "step": 1700
+    },
+    {
+      "epoch": 1.9666210078581212,
+      "grad_norm": 0.1728515625,
+      "learning_rate": 5.384961194988002e-06,
+      "loss": 0.0066,
+      "step": 1705
+    },
+    {
+      "epoch": 1.972388436305962,
+      "grad_norm": 0.138671875,
+      "learning_rate": 5.361672309068182e-06,
+      "loss": 0.0079,
+      "step": 1710
+    },
+    {
+      "epoch": 1.978155864753803,
+      "grad_norm": 0.1640625,
+      "learning_rate": 5.33837553272168e-06,
+      "loss": 0.0077,
+      "step": 1715
+    },
+    {
+      "epoch": 1.9839232932016437,
+      "grad_norm": 0.2177734375,
+      "learning_rate": 5.315071374202792e-06,
+      "loss": 0.0079,
+      "step": 1720
+    },
+    {
+      "epoch": 1.9896907216494846,
+      "grad_norm": 0.111328125,
+      "learning_rate": 5.291760341926873e-06,
+      "loss": 0.0071,
+      "step": 1725
+    },
+    {
+      "epoch": 1.9954581500973254,
+      "grad_norm": 0.1796875,
+      "learning_rate": 5.268442944459234e-06,
+      "loss": 0.0085,
+      "step": 1730
+    },
+    {
+      "epoch": 2.001153485689568,
+      "grad_norm": 0.197265625,
+      "learning_rate": 5.245119690504056e-06,
+      "loss": 0.0073,
+      "step": 1735
+    },
+    {
+      "epoch": 2.006920914137409,
+      "grad_norm": 0.1572265625,
+      "learning_rate": 5.221791088893282e-06,
+      "loss": 0.0078,
+      "step": 1740
+    },
+    {
+      "epoch": 2.0126883425852498,
+      "grad_norm": 0.1552734375,
+      "learning_rate": 5.198457648575533e-06,
+      "loss": 0.008,
+      "step": 1745
+    },
+    {
+      "epoch": 2.0184557710330906,
+      "grad_norm": 0.2255859375,
+      "learning_rate": 5.1751198786049815e-06,
+      "loss": 0.0086,
+      "step": 1750
+    },
+    {
+      "epoch": 2.0242231994809314,
+      "grad_norm": 0.1875,
+      "learning_rate": 5.151778288130267e-06,
+      "loss": 0.0072,
+      "step": 1755
+    },
+    {
+      "epoch": 2.0299906279287723,
+      "grad_norm": 0.14453125,
+      "learning_rate": 5.1284333863833735e-06,
+      "loss": 0.0057,
+      "step": 1760
+    },
+    {
+      "epoch": 2.035758056376613,
+      "grad_norm": 0.15625,
+      "learning_rate": 5.10508568266853e-06,
+      "loss": 0.0056,
+      "step": 1765
+    },
+    {
+      "epoch": 2.041525484824454,
+      "grad_norm": 0.1806640625,
+      "learning_rate": 5.08173568635109e-06,
+      "loss": 0.0061,
+      "step": 1770
+    },
+    {
+      "epoch": 2.047292913272295,
+      "grad_norm": 0.162109375,
+      "learning_rate": 5.058383906846428e-06,
+      "loss": 0.0076,
+      "step": 1775
+    },
+    {
+      "epoch": 2.0530603417201356,
+      "grad_norm": 0.181640625,
+      "learning_rate": 5.035030853608817e-06,
+      "loss": 0.0091,
+      "step": 1780
+    },
+    {
+      "epoch": 2.0588277701679765,
+      "grad_norm": 0.1806640625,
+      "learning_rate": 5.011677036120323e-06,
+      "loss": 0.0062,
+      "step": 1785
+    },
+    {
+      "epoch": 2.0645951986158173,
+      "grad_norm": 0.2080078125,
+      "learning_rate": 4.988322963879679e-06,
+      "loss": 0.0081,
+      "step": 1790
+    },
+    {
+      "epoch": 2.070362627063658,
+      "grad_norm": 0.1875,
+      "learning_rate": 4.964969146391184e-06,
+      "loss": 0.0072,
+      "step": 1795
+    },
+    {
+      "epoch": 2.076130055511499,
+      "grad_norm": 0.224609375,
+      "learning_rate": 4.941616093153574e-06,
+      "loss": 0.0068,
+      "step": 1800
+    },
+    {
+      "epoch": 2.08189748395934,
+      "grad_norm": 0.1767578125,
+      "learning_rate": 4.918264313648911e-06,
+      "loss": 0.0074,
+      "step": 1805
+    },
+    {
+      "epoch": 2.0876649124071807,
+      "grad_norm": 0.2236328125,
+      "learning_rate": 4.894914317331471e-06,
+      "loss": 0.0077,
+      "step": 1810
+    },
+    {
+      "epoch": 2.093432340855021,
+      "grad_norm": 0.162109375,
+      "learning_rate": 4.871566613616626e-06,
+      "loss": 0.0093,
+      "step": 1815
+    },
+    {
+      "epoch": 2.099199769302862,
+      "grad_norm": 0.1796875,
+      "learning_rate": 4.848221711869735e-06,
+      "loss": 0.0078,
+      "step": 1820
+    },
+    {
+      "epoch": 2.1049671977507027,
+      "grad_norm": 0.16796875,
+      "learning_rate": 4.82488012139502e-06,
+      "loss": 0.0071,
+      "step": 1825
+    },
+    {
+      "epoch": 2.1107346261985436,
+      "grad_norm": 0.1953125,
+      "learning_rate": 4.801542351424469e-06,
+      "loss": 0.0072,
+      "step": 1830
+    },
+    {
+      "epoch": 2.1165020546463844,
+      "grad_norm": 0.1533203125,
+      "learning_rate": 4.778208911106718e-06,
+      "loss": 0.0064,
+      "step": 1835
+    },
+    {
+      "epoch": 2.1222694830942253,
+      "grad_norm": 0.14453125,
+      "learning_rate": 4.754880309495946e-06,
+      "loss": 0.0073,
+      "step": 1840
+    },
+    {
+      "epoch": 2.128036911542066,
+      "grad_norm": 0.1943359375,
+      "learning_rate": 4.7315570555407665e-06,
+      "loss": 0.0066,
+      "step": 1845
+    },
+    {
+      "epoch": 2.133804339989907,
+      "grad_norm": 0.1865234375,
+      "learning_rate": 4.7082396580731275e-06,
+      "loss": 0.0059,
+      "step": 1850
+    },
+    {
+      "epoch": 2.1395717684377478,
+      "grad_norm": 0.162109375,
+      "learning_rate": 4.684928625797208e-06,
+      "loss": 0.007,
+      "step": 1855
+    },
+    {
+      "epoch": 2.1453391968855886,
+      "grad_norm": 0.154296875,
+      "learning_rate": 4.661624467278323e-06,
+      "loss": 0.0065,
+      "step": 1860
+    },
+    {
+      "epoch": 2.1511066253334294,
+      "grad_norm": 0.16796875,
+      "learning_rate": 4.63832769093182e-06,
+      "loss": 0.0102,
+      "step": 1865
+    },
+    {
+      "epoch": 2.1568740537812703,
+      "grad_norm": 0.1259765625,
+      "learning_rate": 4.615038805011999e-06,
+      "loss": 0.0063,
+      "step": 1870
+    },
+    {
+      "epoch": 2.162641482229111,
+      "grad_norm": 0.1796875,
+      "learning_rate": 4.591758317601019e-06,
+      "loss": 0.0077,
+      "step": 1875
+    },
+    {
+      "epoch": 2.168408910676952,
+      "grad_norm": 0.1181640625,
+      "learning_rate": 4.568486736597809e-06,
+      "loss": 0.007,
+      "step": 1880
+    },
+    {
+      "epoch": 2.174176339124793,
+      "grad_norm": 0.2041015625,
+      "learning_rate": 4.545224569706994e-06,
+      "loss": 0.0065,
+      "step": 1885
+    },
+    {
+      "epoch": 2.1799437675726336,
+      "grad_norm": 0.1689453125,
+      "learning_rate": 4.521972324427813e-06,
+      "loss": 0.0072,
+      "step": 1890
+    },
+    {
+      "epoch": 2.1857111960204745,
+      "grad_norm": 0.1865234375,
+      "learning_rate": 4.498730508043054e-06,
+      "loss": 0.0076,
+      "step": 1895
+    },
+    {
+      "epoch": 2.1914786244683153,
+      "grad_norm": 0.1455078125,
+      "learning_rate": 4.475499627607981e-06,
+      "loss": 0.0071,
+      "step": 1900
+    },
+    {
+      "epoch": 2.197246052916156,
+      "grad_norm": 0.15234375,
+      "learning_rate": 4.452280189939272e-06,
+      "loss": 0.0078,
+      "step": 1905
+    },
+    {
+      "epoch": 2.203013481363997,
+      "grad_norm": 0.19921875,
+      "learning_rate": 4.429072701603967e-06,
+      "loss": 0.0075,
+      "step": 1910
+    },
+    {
+      "epoch": 2.208780909811838,
+      "grad_norm": 0.28125,
+      "learning_rate": 4.405877668908412e-06,
+      "loss": 0.0069,
+      "step": 1915
+    },
+    {
+      "epoch": 2.2145483382596787,
+      "grad_norm": 0.185546875,
+      "learning_rate": 4.382695597887212e-06,
+      "loss": 0.01,
+      "step": 1920
+    },
+    {
+      "epoch": 2.220315766707519,
+      "grad_norm": 0.1865234375,
+      "learning_rate": 4.359526994292202e-06,
+      "loss": 0.008,
+      "step": 1925
+    },
+    {
+      "epoch": 2.22608319515536,
+      "grad_norm": 0.162109375,
+      "learning_rate": 4.336372363581391e-06,
+      "loss": 0.0069,
+      "step": 1930
+    },
+    {
+      "epoch": 2.2318506236032007,
+      "grad_norm": 0.20703125,
+      "learning_rate": 4.313232210907959e-06,
+      "loss": 0.0079,
+      "step": 1935
+    },
+    {
+      "epoch": 2.2376180520510416,
+      "grad_norm": 0.1962890625,
+      "learning_rate": 4.290107041109224e-06,
+      "loss": 0.0077,
+      "step": 1940
+    },
+    {
+      "epoch": 2.2433854804988824,
+      "grad_norm": 0.205078125,
+      "learning_rate": 4.26699735869563e-06,
+      "loss": 0.0074,
+      "step": 1945
+    },
+    {
+      "epoch": 2.2491529089467233,
+      "grad_norm": 0.205078125,
+      "learning_rate": 4.243903667839735e-06,
+      "loss": 0.0084,
+      "step": 1950
+    },
+    {
+      "epoch": 2.254920337394564,
+      "grad_norm": 0.19140625,
+      "learning_rate": 4.2208264723652234e-06,
+      "loss": 0.0073,
+      "step": 1955
+    },
+    {
+      "epoch": 2.260687765842405,
+      "grad_norm": 0.169921875,
+      "learning_rate": 4.197766275735907e-06,
+      "loss": 0.0084,
+      "step": 1960
+    },
+    {
+      "epoch": 2.266455194290246,
+      "grad_norm": 0.2275390625,
+      "learning_rate": 4.174723581044742e-06,
+      "loss": 0.008,
+      "step": 1965
+    },
+    {
+      "epoch": 2.2722226227380866,
+      "grad_norm": 0.171875,
+      "learning_rate": 4.151698891002848e-06,
+      "loss": 0.0072,
+      "step": 1970
+    },
+    {
+      "epoch": 2.2779900511859275,
+      "grad_norm": 0.1376953125,
+      "learning_rate": 4.128692707928553e-06,
+      "loss": 0.0072,
+      "step": 1975
+    },
+    {
+      "epoch": 2.2837574796337683,
+      "grad_norm": 0.189453125,
+      "learning_rate": 4.105705533736424e-06,
+      "loss": 0.0061,
+      "step": 1980
+    },
+    {
+      "epoch": 2.289524908081609,
+      "grad_norm": 0.1826171875,
+      "learning_rate": 4.082737869926326e-06,
+      "loss": 0.0113,
+      "step": 1985
+    },
+    {
+      "epoch": 2.29529233652945,
+      "grad_norm": 0.162109375,
+      "learning_rate": 4.0597902175724664e-06,
+      "loss": 0.0069,
+      "step": 1990
+    },
+    {
+      "epoch": 2.301059764977291,
+      "grad_norm": 0.162109375,
+      "learning_rate": 4.036863077312479e-06,
+      "loss": 0.0075,
+      "step": 1995
+    },
+    {
+      "epoch": 2.3068271934251317,
+      "grad_norm": 0.1826171875,
+      "learning_rate": 4.013956949336494e-06,
+      "loss": 0.0068,
+      "step": 2000
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 3468,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 4,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.6264730197004124e+18,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoints/sft3b_lr1e5_bs64/checkpoint-2500/added_tokens.json

@@ -0,0 +1,35 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|action_sep|>": 151670,
+  "<|arg_sep|>": 151671,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|call_sep|>": 151666,
+  "<|end_of_text|>": 151673,
+  "<|endoftext|>": 151643,
+  "<|exception_sep|>": 151669,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|frame_sep|>": 151672,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|latent_end|>": 151675,
+  "<|latent_start|>": 151674,
+  "<|line_sep|>": 151667,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|return_sep|>": 151668,
+  "<|trace_context_start|>": 151665,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoints/sft3b_lr1e5_bs64/checkpoint-2500/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

checkpoints/sft3b_lr1e5_bs64/checkpoint-2500/config.json

@@ -0,0 +1,66 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "dtype": "bfloat16",
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 36,
+  "model_type": "qwen2",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 36,
+  "num_key_value_heads": 2,
+  "pad_token_id": 151643,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "transformers_version": "4.57.6",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151676
+}

checkpoints/sft3b_lr1e5_bs64/checkpoint-2500/generation_config.json

@@ -0,0 +1,8 @@
+{
+  "eos_token_id": [
+    151643
+  ],
+  "max_new_tokens": 2048,
+  "pad_token_id": 151643,
+  "transformers_version": "4.57.6"
+}

checkpoints/sft3b_lr1e5_bs64/checkpoint-2500/model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57c483a5461166129b278440d51b9d96ae1efcad4b9f2267b2f6f1aabfa191ad
+size 4956495344

checkpoints/sft3b_lr1e5_bs64/checkpoint-2500/model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9e047c3ecb224fbaaae0df9340ebbfd4b7d8b394e8185855391e0f4402b99fa
+size 1214366696

checkpoints/sft3b_lr1e5_bs64/checkpoint-2500/model.safetensors.index.json

@@ -0,0 +1,442 @@
+{
+  "metadata": {
+    "total_parameters": 3085406208,
+    "total_size": 6170812416
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.28.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.28.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

checkpoints/sft3b_lr1e5_bs64/checkpoint-2500/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoints/sft3b_lr1e5_bs64/checkpoint-2500/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83a790d654474f5dfe225f889afd0210313eb1083f942671f2c4b8e95a1c922b
+size 11424004