diff --git a/.gitattributes b/.gitattributes index bc18b3d16f03cc375da9ed9642078b7dedf57302..f195a9debddcc2c691a7e80bfbf554fb6251c39d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -56,3 +56,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text 26_09_08_sft_email_v4/export/gemma4/120/tokenizer.json filter=lfs diff=lfs merge=lfs -text 26_09_08_sft_email_v4/export/gemma4/180/tokenizer.json filter=lfs diff=lfs merge=lfs -text 26_09_08_sft_email_v4/export/gemma4/60/tokenizer.json filter=lfs diff=lfs merge=lfs -text +26_09_08_sft_email_v4_v3/export/gemma4/140/tokenizer.json filter=lfs diff=lfs merge=lfs -text +26_09_08_sft_email_v4_v3/export/gemma4/210/tokenizer.json filter=lfs diff=lfs merge=lfs -text +26_09_08_sft_email_v4_v3/export/gemma4/70/tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/26_09_08_sft_email_v4_v3/config.yaml b/26_09_08_sft_email_v4_v3/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..773420cdd9fc16848951d31b780c22ee4a325319 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/config.yaml @@ -0,0 +1,267 @@ +data: + datasets: + training: + datasets: + - path: /mnt/queue1/shashank/exp/data/26_09_04_sft_email_v3/fastllm/fast_llm_config.yaml + type: file + - path: /mnt/queue1/shashank/exp/data/26_09_08_sft_email_v4/fastllm/fast_llm_config.yaml + type: file + - path: /mnt/queue1/shruthan/data/gemma_sft_replay/10k/fastllm/fast_llm_config.yaml + type: file + name: gemma_replay_and_automation_bench + type: blended + weights: + - 0.15008542238000852 + - 0.3285671651047079 + - 0.5213474125152836 + maximum_document_length: 32768 + micro_batch_size: 32768 + truncate_documents: false +model: + base_model: + decoder: + blocks: + full_attention: + mixer: + add_linear_biases: false + dropout: 0.0 + head_groups: 2 + head_size: 512 + heads: 16 + implementation: sdpa + key_norm: + epsilon: 1.0e-06 + type: rms_norm + query_norm: + epsilon: 1.0e-06 + type: rms_norm + rotary: + partial_rotary_factor: 0.25 + theta: 1000000.0 + type: proportional + shared_key_value: true + softmax_scale_power: 0.0 + type: attention + value_norm: + epsilon: 1.0e-06 + type: fixed_rms_norm + mlp: + dense: + activation: gelu + add_linear_biases: false + gated: true + intermediate_size: 2112 + post_norm: + epsilon: 1.0e-06 + type: rms_norm + pre_norm: + epsilon: 1.0e-06 + type: rms_norm + recompute_level: full + type: mlp + routed: + activation: gelu + add_linear_biases: false + auxiliary_loss_coefficient: 0.0 + experts: 128 + experts_per_token: 8 + gated: true + intermediate_size: 704 + post_norm: + epsilon: 1.0e-06 + type: rms_norm + pre_norm: + epsilon: 1.0e-06 + type: rms_norm + recompute_level: full + router_input_scale: 0.018844459036110227 + router_normalization: + epsilon: 1.0e-06 + type: fixed_rms_norm + router_per_expert_scale: + enabled: true + router_scale: + enabled: true + type: moe + type: hybrid_moe + normalization: + epsilon: 1.0e-06 + type: rms_norm + output_scale: + enabled: true + lr_scale: 0.0 + post_mixer_normalization: + epsilon: 1.0e-06 + type: rms_norm + post_mlp_normalization: + epsilon: 1.0e-06 + type: rms_norm + pre_mlp_normalization: + type: none + type: decoder + sliding_attention: + mixer: + add_linear_biases: false + dropout: 0.0 + head_groups: 8 + head_size: 256 + heads: 16 + implementation: flash + key_norm: + epsilon: 1.0e-06 + type: rms_norm + query_norm: + epsilon: 1.0e-06 + type: rms_norm + rotary: + theta: 10000.0 + type: default + softmax_scale_power: 0.0 + type: attention + value_norm: + epsilon: 1.0e-06 + type: fixed_rms_norm + window_size: 1024 + mlp: + dense: + activation: gelu + add_linear_biases: false + gated: true + intermediate_size: 2112 + post_norm: + epsilon: 1.0e-06 + type: rms_norm + pre_norm: + epsilon: 1.0e-06 + type: rms_norm + recompute_level: full + type: mlp + routed: + activation: gelu + add_linear_biases: false + auxiliary_loss_coefficient: 0.0 + experts: 128 + experts_per_token: 8 + gated: true + intermediate_size: 704 + post_norm: + epsilon: 1.0e-06 + type: rms_norm + pre_norm: + epsilon: 1.0e-06 + type: rms_norm + recompute_level: full + router_input_scale: 0.018844459036110227 + router_normalization: + epsilon: 1.0e-06 + type: fixed_rms_norm + router_per_expert_scale: + enabled: true + router_scale: + enabled: true + type: moe + type: hybrid_moe + normalization: + epsilon: 1.0e-06 + type: rms_norm + output_scale: + enabled: true + lr_scale: 0.0 + post_mixer_normalization: + epsilon: 1.0e-06 + type: rms_norm + post_mlp_normalization: + epsilon: 1.0e-06 + type: rms_norm + pre_mlp_normalization: + type: none + type: decoder + num_blocks: 30 + pattern: + - sliding_attention + - sliding_attention + - sliding_attention + - sliding_attention + - sliding_attention + - full_attention + - sliding_attention + - sliding_attention + - sliding_attention + - sliding_attention + - sliding_attention + - full_attention + - sliding_attention + - sliding_attention + - sliding_attention + - sliding_attention + - sliding_attention + - full_attention + - sliding_attention + - sliding_attention + - sliding_attention + - sliding_attention + - sliding_attention + - full_attention + - sliding_attention + - sliding_attention + - sliding_attention + - sliding_attention + - sliding_attention + - full_attention + type: pattern + embeddings: + embedding_scale: 53.0659966456864 + vocab_parallel: false + vocab_size: 262144 + head: + cross_entropy_splits: 8 + final_logit_softcap: 30.0 + normalization: + epsilon: 1.0e-06 + type: rms_norm + hidden_size: 2816 + peft: + type: none + tied_embedding_weight: true + distributed: + compute_dtype: bfloat16 + local_world_size: 8 + rank: 0 + sequence_data_parallel: 4 + sequence_tensor_parallel: true + tensor_parallel: 2 + timeout: 3600.0 + world_size: 32 + multi_stage: + zero_stage: 3 + type: gpt +optimizer: + learning_rate: + base: 2.0e-05 + decay_iterations: 210 + decay_style: cosine + warmup_iterations: 7 + weight_decay: 0.01 +pretrained: + format: gemma4 + load_config: model + model_weights: true + path: /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm +run: + experiment_dir: /mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3 +schedule: + breadth_first_micro_batches: 1 + depth_first_micro_batches: 16 +training: + evaluators: {} + export: + format: gemma4 + interval: 70 + logs: + interval: 1 + num_workers: 0 + train_iters: 210 + wandb: + entity_name: corellm + project_name: gemma-sft +type: train_gpt diff --git a/26_09_08_sft_email_v4_v3/config_verbose.yaml b/26_09_08_sft_email_v4_v3/config_verbose.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dea95960ca01f8ae9a1b6d5bb4a43a484b2f654b --- /dev/null +++ b/26_09_08_sft_email_v4_v3/config_verbose.yaml @@ -0,0 +1,910 @@ +callbacks: {} +data: + data_sample_warn_time_ms: 1000.0 + datasets: + training: + datasets: + - path: /mnt/queue1/shashank/exp/data/26_09_04_sft_email_v3/fastllm/fast_llm_config.yaml + type: file + - path: /mnt/queue1/shashank/exp/data/26_09_08_sft_email_v4/fastllm/fast_llm_config.yaml + type: file + - path: /mnt/queue1/shruthan/data/gemma_sft_replay/10k/fastllm/fast_llm_config.yaml + type: file + name: gemma_replay_and_automation_bench + type: blended + weights: + - 0.15008542238000852 + - 0.3285671651047079 + - 0.5213474125152836 + gpu: true + maximum_document_length: 32768 + micro_batch_size: 32768 + multiprocessing_context: spawn + seed: 784569 + shuffle: epoch + truncate_documents: false + type: null +model: + base_model: + decoder: + blocks: + full_attention: + distillation_loss_weight: 1.0 + distillation_model: null + dropout: 0.0 + lr_scale: null + mixer: + add_linear_biases: false + causal: true + dense_layer: + apply_peft: null + bias: + enabled: null + initialization: + type: null + lr_scale: null + type: null + lr_scale: null + type: null + weight: + initialization: + type: null + lr_scale: null + type: null + dropout: 0.0 + head_groups: 2 + head_size: 512 + heads: 16 + implementation: sdpa + key_layer: + apply_peft: null + bias: + enabled: null + initialization: + type: null + lr_scale: null + type: null + lr_scale: null + type: null + weight: + initialization: + type: null + lr_scale: null + type: null + key_norm: + epsilon: 1.0e-06 + implementation: auto + lr_scale: null + type: rms_norm + weight: + initialization: + type: null + lr_scale: null + type: null + zero_centered: false + lr_scale: null + query_layer: + apply_peft: null + bias: + enabled: null + initialization: + type: null + lr_scale: null + type: null + lr_scale: null + type: null + weight: + initialization: + type: null + lr_scale: null + type: null + query_norm: + epsilon: 1.0e-06 + implementation: auto + lr_scale: null + type: rms_norm + weight: + initialization: + type: null + lr_scale: null + type: null + zero_centered: false + rotary: + partial_rotary_factor: 0.25 + theta: 1000000.0 + type: proportional + shared_key_value: true + softmax_scale_power: 0.0 + type: attention + use_flash_attention: true + value_layer: + apply_peft: null + bias: + enabled: null + initialization: + type: null + lr_scale: null + type: null + lr_scale: null + type: null + weight: + initialization: + type: null + lr_scale: null + type: null + value_norm: + epsilon: 1.0e-06 + lr_scale: null + type: fixed_rms_norm + window_size: null + mlp: + dense: + activation: gelu + add_linear_biases: false + gated: true + intermediate_size: 2112 + layer_1: + apply_peft: null + bias: + enabled: null + initialization: + type: null + lr_scale: null + type: null + lr_scale: null + type: null + weight: + initialization: + type: null + lr_scale: null + type: null + layer_2: + apply_peft: null + bias: + enabled: null + initialization: + type: null + lr_scale: null + type: null + lr_scale: null + type: null + weight: + initialization: + type: null + lr_scale: null + type: null + lr_scale: null + post_norm: + epsilon: 1.0e-06 + implementation: auto + lr_scale: null + type: rms_norm + weight: + initialization: + type: null + lr_scale: null + type: null + zero_centered: false + pre_norm: + epsilon: 1.0e-06 + implementation: auto + lr_scale: null + type: rms_norm + weight: + initialization: + type: null + lr_scale: null + type: null + zero_centered: false + recompute_level: full + type: mlp + lr_scale: null + post_norm: null + pre_norm: null + routed: + activation: gelu + add_linear_biases: false + auxiliary_loss_coefficient: 0.0 + dropless_dynamic_shape: false + experts: 128 + experts_per_token: 8 + gated: true + implementation: auto + intermediate_size: 704 + jitter_eps: 0.0 + layer_1: + apply_peft: null + bias: + enabled: null + initialization: + type: null + lr_scale: null + type: null + lr_scale: null + type: null + weight: + initialization: + type: null + lr_scale: null + type: null + layer_2: + apply_peft: null + bias: + enabled: null + initialization: + type: null + lr_scale: null + type: null + lr_scale: null + type: null + weight: + initialization: + type: null + lr_scale: null + type: null + lr_scale: null + post_norm: + epsilon: 1.0e-06 + implementation: auto + lr_scale: null + type: rms_norm + weight: + initialization: + type: null + lr_scale: null + type: null + zero_centered: false + pre_norm: + epsilon: 1.0e-06 + implementation: auto + lr_scale: null + type: rms_norm + weight: + initialization: + type: null + lr_scale: null + type: null + zero_centered: false + recompute_level: full + router: + apply_peft: null + lr_scale: null + type: null + weight: + initialization: + type: null + lr_scale: null + type: null + router_input_scale: 0.018844459036110227 + router_normalization: + epsilon: 1.0e-06 + lr_scale: null + type: fixed_rms_norm + router_per_expert_scale: + enabled: true + initialization: + type: null + lr_scale: null + type: null + router_scale: + enabled: true + initialization: + type: null + lr_scale: null + type: null + routing: aux_loss + shared_experts: 0 + type: moe + z_loss_coefficient: 0.0 + type: hybrid_moe + normalization: + epsilon: 1.0e-06 + implementation: auto + lr_scale: null + type: rms_norm + weight: + initialization: + type: null + lr_scale: null + type: null + zero_centered: false + output_scale: + enabled: true + initialization: + type: null + lr_scale: 0.0 + type: null + post_mixer_normalization: + epsilon: 1.0e-06 + implementation: auto + lr_scale: null + type: rms_norm + weight: + initialization: + type: null + lr_scale: null + type: null + zero_centered: false + post_mlp_normalization: + epsilon: 1.0e-06 + implementation: auto + lr_scale: null + type: rms_norm + weight: + initialization: + type: null + lr_scale: null + type: null + zero_centered: false + pre_mixer_normalization: null + pre_mlp_normalization: + lr_scale: null + type: none + type: decoder + sliding_attention: + distillation_loss_weight: 1.0 + distillation_model: null + dropout: 0.0 + lr_scale: null + mixer: + add_linear_biases: false + causal: true + dense_layer: + apply_peft: null + bias: + enabled: null + initialization: + type: null + lr_scale: null + type: null + lr_scale: null + type: null + weight: + initialization: + type: null + lr_scale: null + type: null + dropout: 0.0 + head_groups: 8 + head_size: 256 + heads: 16 + implementation: flash + key_layer: + apply_peft: null + bias: + enabled: null + initialization: + type: null + lr_scale: null + type: null + lr_scale: null + type: null + weight: + initialization: + type: null + lr_scale: null + type: null + key_norm: + epsilon: 1.0e-06 + implementation: auto + lr_scale: null + type: rms_norm + weight: + initialization: + type: null + lr_scale: null + type: null + zero_centered: false + lr_scale: null + query_layer: + apply_peft: null + bias: + enabled: null + initialization: + type: null + lr_scale: null + type: null + lr_scale: null + type: null + weight: + initialization: + type: null + lr_scale: null + type: null + query_norm: + epsilon: 1.0e-06 + implementation: auto + lr_scale: null + type: rms_norm + weight: + initialization: + type: null + lr_scale: null + type: null + zero_centered: false + rotary: + theta: 10000.0 + type: default + shared_key_value: false + softmax_scale_power: 0.0 + type: attention + use_flash_attention: true + value_layer: + apply_peft: null + bias: + enabled: null + initialization: + type: null + lr_scale: null + type: null + lr_scale: null + type: null + weight: + initialization: + type: null + lr_scale: null + type: null + value_norm: + epsilon: 1.0e-06 + lr_scale: null + type: fixed_rms_norm + window_size: 1024 + mlp: + dense: + activation: gelu + add_linear_biases: false + gated: true + intermediate_size: 2112 + layer_1: + apply_peft: null + bias: + enabled: null + initialization: + type: null + lr_scale: null + type: null + lr_scale: null + type: null + weight: + initialization: + type: null + lr_scale: null + type: null + layer_2: + apply_peft: null + bias: + enabled: null + initialization: + type: null + lr_scale: null + type: null + lr_scale: null + type: null + weight: + initialization: + type: null + lr_scale: null + type: null + lr_scale: null + post_norm: + epsilon: 1.0e-06 + implementation: auto + lr_scale: null + type: rms_norm + weight: + initialization: + type: null + lr_scale: null + type: null + zero_centered: false + pre_norm: + epsilon: 1.0e-06 + implementation: auto + lr_scale: null + type: rms_norm + weight: + initialization: + type: null + lr_scale: null + type: null + zero_centered: false + recompute_level: full + type: mlp + lr_scale: null + post_norm: null + pre_norm: null + routed: + activation: gelu + add_linear_biases: false + auxiliary_loss_coefficient: 0.0 + dropless_dynamic_shape: false + experts: 128 + experts_per_token: 8 + gated: true + implementation: auto + intermediate_size: 704 + jitter_eps: 0.0 + layer_1: + apply_peft: null + bias: + enabled: null + initialization: + type: null + lr_scale: null + type: null + lr_scale: null + type: null + weight: + initialization: + type: null + lr_scale: null + type: null + layer_2: + apply_peft: null + bias: + enabled: null + initialization: + type: null + lr_scale: null + type: null + lr_scale: null + type: null + weight: + initialization: + type: null + lr_scale: null + type: null + lr_scale: null + post_norm: + epsilon: 1.0e-06 + implementation: auto + lr_scale: null + type: rms_norm + weight: + initialization: + type: null + lr_scale: null + type: null + zero_centered: false + pre_norm: + epsilon: 1.0e-06 + implementation: auto + lr_scale: null + type: rms_norm + weight: + initialization: + type: null + lr_scale: null + type: null + zero_centered: false + recompute_level: full + router: + apply_peft: null + lr_scale: null + type: null + weight: + initialization: + type: null + lr_scale: null + type: null + router_input_scale: 0.018844459036110227 + router_normalization: + epsilon: 1.0e-06 + lr_scale: null + type: fixed_rms_norm + router_per_expert_scale: + enabled: true + initialization: + type: null + lr_scale: null + type: null + router_scale: + enabled: true + initialization: + type: null + lr_scale: null + type: null + routing: aux_loss + shared_experts: 0 + type: moe + z_loss_coefficient: 0.0 + type: hybrid_moe + normalization: + epsilon: 1.0e-06 + implementation: auto + lr_scale: null + type: rms_norm + weight: + initialization: + type: null + lr_scale: null + type: null + zero_centered: false + output_scale: + enabled: true + initialization: + type: null + lr_scale: 0.0 + type: null + post_mixer_normalization: + epsilon: 1.0e-06 + implementation: auto + lr_scale: null + type: rms_norm + weight: + initialization: + type: null + lr_scale: null + type: null + zero_centered: false + post_mlp_normalization: + epsilon: 1.0e-06 + implementation: auto + lr_scale: null + type: rms_norm + weight: + initialization: + type: null + lr_scale: null + type: null + zero_centered: false + pre_mixer_normalization: null + pre_mlp_normalization: + lr_scale: null + type: none + type: decoder + lr_scale: null + num_blocks: 30 + pattern: + - sliding_attention + - sliding_attention + - sliding_attention + - sliding_attention + - sliding_attention + - full_attention + - sliding_attention + - sliding_attention + - sliding_attention + - sliding_attention + - sliding_attention + - full_attention + - sliding_attention + - sliding_attention + - sliding_attention + - sliding_attention + - sliding_attention + - full_attention + - sliding_attention + - sliding_attention + - sliding_attention + - sliding_attention + - sliding_attention + - full_attention + - sliding_attention + - sliding_attention + - sliding_attention + - sliding_attention + - sliding_attention + - full_attention + type: pattern + embeddings: + dropout: 0.0 + embedding_scale: 53.0659966456864 + full_precision_residual: false + lr_scale: null + num_position_embeddings: 2048 + position_embeddings: + enabled: null + initialization: + type: null + lr_scale: null + type: null + type: null + vocab_parallel: false + vocab_size: 262144 + word_embeddings: + initialization: + type: null + lr_scale: null + type: null + head: + cross_entropy_splits: 8 + final_logit_softcap: 30.0 + logits_scale_factor: 1.0 + losses: {} + lr_scale: null + normalization: + epsilon: 1.0e-06 + implementation: auto + lr_scale: null + type: rms_norm + weight: + initialization: + type: null + lr_scale: null + type: null + zero_centered: false + output_weight: + initialization: + type: null + lr_scale: null + type: null + prediction_heads: 1 + prediction_loss_coefficient: null + type: null + hidden_size: 2816 + lr_scale: null + peft: + type: none + tied_embedding_weight: true + type: null + use_megatron_initialization: false + distributed: + backend: nccl + compute_dtype: bfloat16 + dp_seed_shift: 317767863445754100075399033823 + force_cpu_initialization: false + inference_seed_shift: 220111337975202516901860145957 + local_world_size: 8 + optimization_dtype: float32 + pipeline_first: false + pipeline_parallel: 1 + pp_gen_init_seed_shift: 631112027069964424381239824623 + pp_gen_seed_shift: 500460795349110443334056239993 + pp_seed_shift: 811026271858507938190098775099 + rank: 0 + reproducible_init: false + sample_seed_shift: 751127116949963770353413160199 + seed: 1234 + sequence_data_parallel: 4 + sequence_tensor_parallel: true + tensor_parallel: 2 + timeout: 3600.0 + tp_gen_init_seed_shift: 894750739684993243926471979237 + tp_gen_seed_shift: 278779420836085904093221202933 + tp_seed_shift: 705275193289568515128435800471 + train_seed_shift: 938219878163699459065752841447 + type: null + use_cuda: true + valid_seed_shift: 683552447587140661489672773353 + world_size: 32 + multi_stage: + debug_activation_memory: false + debug_all_param_gradients: 0 + debug_global_tensors: true + debug_layer_gradients: 0 + debug_layer_outputs: 0 + debug_param_gradients: 0 + debug_param_init: 0 + debug_param_update: 0 + debug_tensor_parallel: false + full_precision_gradients: true + layers_per_stage: 1.0 + num_grad_buffers: 2 + num_weight_buffers: 2 + pipeline_delay: 0.0 + store_frozen_weights_in_optimization_precision: true + type: null + zero_stage: 3 + type: gpt +optimizer: + beta_1: 0.9 + beta_2: 0.999 + default_learning_rate_scale: 1.0 + epsilon: 1.0e-08 + gradient_norm_clipping: 1.0 + gradient_scaler: + constant: null + hysteresis: 2 + initial: 65536.0 + minimum: 1.0 + type: null + window: 1000 + learning_rate: + base: 2.0e-05 + decay_iterations: 210 + decay_power: 1.0 + decay_style: cosine + minimum: 0.0 + type: null + warmup_iterations: 7 + type: null + weight_decay: 0.01 +pretrained: + format: gemma4 + load_config: model + model_weights: true + optimizer_state: false + path: /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm + timeout: null + type: null +profiling: + averages: false + cpu: false + cuda: false + cycles: 1 + export: false + log: false + ranks: [] + skip: 1 + table_width: 80 + trace: false + type: null + wait: 0 + warmup: 1 +reference_models: {} +run: + enable_all_loggers: false + enable_triton_kernels: true + experiment_dir: /mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3 + experiment_name: null + log_timestamps: true + model_debug_level: 0 + structured_logs: true + tensor_logs: + full_tensors: false + max_elements: 8 + save: false + show: true + type: null + torch_dynamo_enable: true + triton_linear_kernels: false + type: null +schedule: + breadth_first_micro_batches: 1 + data_batch_warn_time_ms: 1000.0 + data_overlap: true + debug_schedule: false + debug_send_recv: false + depth_first_micro_batches: 16 + micro_batch_splits: 1 + pipeline_overlap: true + profile_schedule: false + skip_step: false + throttle_cpu: true + throttle_cpu_delay: 1 + throttle_cpu_rate: 1 + type: null +training: + checkpoint: + interval: null + keep: 5 + keep_every: null + offset: 0 + type: null + evaluators: {} + export: + data_type: null + format: gemma4 + interval: 70 + keep: null + keep_every: null + model_weights: true + offset: 0 + optimizer_state: false + parameters_per_file: 4294967296 + type: null + logs: + interval: 1 + offset: 0 + type: null + num_workers: 0 + prefetch_factor: null + shutdown: + interval: null + offset: 0 + type: null + timeout: 3600.0 + train_iters: 210 + type: null + wandb: + alert: + interval: null + offset: 0 + status_updates: null + type: null + entity_name: corellm + group_name: default + project_name: gemma-sft + type: null +type: train_gpt diff --git a/26_09_08_sft_email_v4_v3/configure_fastllm_model_config.py b/26_09_08_sft_email_v4_v3/configure_fastllm_model_config.py new file mode 100644 index 0000000000000000000000000000000000000000..7362263b308597ef88e93300d28374c571ce5b37 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/configure_fastllm_model_config.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Fill in the data-dependent training knobs of a fastllm model_config.yaml. + +DATASETS (below) lists the sources to blend, each a `fast_llm_config.yaml` written by the +`fast-llm prepare gpt_memmap` tokenization step (that file carries the source's real +`metadata.num_tokens`). This script writes that list into model_config.yaml's +data.datasets.training.datasets and derives the values that depend on their token counts: + + data.datasets.training.datasets DATASETS, as {type: file, path: ...} entries + data.datasets.training.weights token count of each source (token-proportional blend) + schedule.depth_first_micro_batches solved from TARGET_GLOBAL_BATCH_SIZE / NUM_NODES + training.train_iters NUM_EPOCHS * num_steps_per_epoch + training.checkpoint.interval num_steps_per_epoch + training.export.interval num_steps_per_epoch + optimizer.learning_rate.warmup_iterations num_steps_per_epoch // 10 + optimizer.learning_rate.decay_iterations training.train_iters + +where, with TP=2 and SDP=4 already fixed in the model config, a single node contributes +`depth_first_micro_batches` sequences per optimizer step (SDP/TP split one sequence across +ranks rather than processing several in parallel), so: + + global_batch_size = NUM_NODES * depth_first_micro_batches + num_steps_per_epoch = ceil(EXTRA_STEPS_FRACTION-padded total_tokens / (global_batch_size * seq_len)), + then rounded up to the next multiple of 10 + +seq_len is read from the model config's data.maximum_document_length. total_tokens is the sum +of every source's num_tokens. The 5% padding accounts for packing-efficiency slack -- exact +step counts are not expected to land precisely on an epoch boundary. + +All run configuration lives in the CONFIGURATION block below -- edit it directly rather than +passing command-line arguments. +""" + +from __future__ import annotations + +import math +from pathlib import Path + +import yaml + +# ----------------------------------------------------------------------------- +# Configuration -- edit these instead of passing CLI arguments. +# ----------------------------------------------------------------------------- + +MODEL_CONFIG_PATH = Path("/mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3/model_config.yaml") + +# fast_llm_config.yaml path for each source to blend into training +DATASETS = [ + "/mnt/queue1/shashank/exp/data/26_09_04_sft_email_v3/fastllm/fast_llm_config.yaml", + "/mnt/queue1/shashank/exp/data/26_09_08_sft_email_v4/fastllm/fast_llm_config.yaml", + "/mnt/queue1/shruthan/data/gemma_sft_replay/10k/fastllm/fast_llm_config.yaml", +] + +NUM_NODES = 4 +TARGET_GLOBAL_BATCH_SIZE = 64 +NUM_EPOCHS = 3 +EXTRA_STEPS_FRACTION = 0.05 # padding for packing-efficiency slack + + +def load_yaml(path: Path) -> dict: + with path.open("r", encoding="utf-8") as f: + return yaml.safe_load(f) + + +def source_num_tokens(fast_llm_config_path: str) -> int: + source_config = load_yaml(Path(fast_llm_config_path)) + return source_config["metadata"]["num_tokens"] + + +def main() -> int: + config = load_yaml(MODEL_CONFIG_PATH) + + seq_len = config["data"]["maximum_document_length"] + + token_counts = [source_num_tokens(path) for path in DATASETS] + total_tokens = sum(token_counts) + + depth_first_micro_batches = max(1, round(TARGET_GLOBAL_BATCH_SIZE / NUM_NODES)) + global_batch_size = NUM_NODES * depth_first_micro_batches + + num_steps_per_epoch = math.ceil((1.0 + EXTRA_STEPS_FRACTION) * total_tokens / (global_batch_size * seq_len)) + num_steps_per_epoch = math.ceil(num_steps_per_epoch / 10) * 10 + train_iters = NUM_EPOCHS * num_steps_per_epoch + warmup_iterations = num_steps_per_epoch // 10 + + config["data"]["datasets"]["training"]["datasets"] = [{"type": "file", "path": path} for path in DATASETS] + config["data"]["datasets"]["training"]["weights"] = token_counts + config["schedule"]["depth_first_micro_batches"] = depth_first_micro_batches + config["training"]["train_iters"] = train_iters + if "checkpoint" in config["training"]: + config["training"]["checkpoint"]["interval"] = num_steps_per_epoch + config["training"]["export"]["interval"] = num_steps_per_epoch + config["optimizer"]["learning_rate"]["warmup_iterations"] = warmup_iterations + config["optimizer"]["learning_rate"]["decay_iterations"] = train_iters + + with MODEL_CONFIG_PATH.open("w", encoding="utf-8") as f: + yaml.safe_dump(config, f, sort_keys=False) + + print(f"weights: {token_counts} (total_tokens={total_tokens}, seq_len={seq_len})") + print(f"global_batch_size={global_batch_size} depth_first_micro_batches={depth_first_micro_batches}") + print(f"num_steps_per_epoch={num_steps_per_epoch} train_iters={train_iters} warmup_iterations={warmup_iterations}") + print(f"Updated {MODEL_CONFIG_PATH}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/26_09_08_sft_email_v4_v3/dataset_cache/__mnt__queue1__shashank__exp__data__26_09_04_sft_email_v3__fastllm__shard_0_0.fast_llm_dataset_ns_2019_sl_32768_s_784569_shuffled_cumsum.npy b/26_09_08_sft_email_v4_v3/dataset_cache/__mnt__queue1__shashank__exp__data__26_09_04_sft_email_v3__fastllm__shard_0_0.fast_llm_dataset_ns_2019_sl_32768_s_784569_shuffled_cumsum.npy new file mode 100644 index 0000000000000000000000000000000000000000..40c725edef25aea780c5400931e1a6bcd2311e0c --- /dev/null +++ b/26_09_08_sft_email_v4_v3/dataset_cache/__mnt__queue1__shashank__exp__data__26_09_04_sft_email_v3__fastllm__shard_0_0.fast_llm_dataset_ns_2019_sl_32768_s_784569_shuffled_cumsum.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f78708060f6aca29e7c433ff174ad0b6a4c9d233f2ced9e56368a030a0ad1ea +size 10504 diff --git a/26_09_08_sft_email_v4_v3/dataset_cache/__mnt__queue1__shashank__exp__data__26_09_04_sft_email_v3__fastllm__shard_0_0.fast_llm_dataset_ns_2019_sl_32768_s_784569_shuffling.npy b/26_09_08_sft_email_v4_v3/dataset_cache/__mnt__queue1__shashank__exp__data__26_09_04_sft_email_v3__fastllm__shard_0_0.fast_llm_dataset_ns_2019_sl_32768_s_784569_shuffling.npy new file mode 100644 index 0000000000000000000000000000000000000000..cf47d347801b702098f35f44f77508849b772615 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/dataset_cache/__mnt__queue1__shashank__exp__data__26_09_04_sft_email_v3__fastllm__shard_0_0.fast_llm_dataset_ns_2019_sl_32768_s_784569_shuffling.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1dddd9262b742465744ed1deceacdbf3468f0d932dd78c7e2eba3e5ccb4d7c14 +size 26088 diff --git a/26_09_08_sft_email_v4_v3/dataset_cache/__mnt__queue1__shashank__exp__data__26_09_04_sft_email_v3__fastllm__shard_0_0.yaml b/26_09_08_sft_email_v4_v3/dataset_cache/__mnt__queue1__shashank__exp__data__26_09_04_sft_email_v3__fastllm__shard_0_0.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc24901ea53aa96a3b72fa502a6a579823d1cd73 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/dataset_cache/__mnt__queue1__shashank__exp__data__26_09_04_sft_email_v3__fastllm__shard_0_0.yaml @@ -0,0 +1,20 @@ +config: + dataset_name: training + gpu: true + maximum_document_length: 32768 + micro_batch_size: 32768 + predicted_tokens: 1 + shuffle: epoch + token_cumsum_rate: 10 + truncate_documents: false + type: null + world_size: 32 +dataset: + documents_per_epoch: 3947 + name: __mnt__queue1__shashank__exp__data__26_09_04_sft_email_v3__fastllm__shard_0_0.fast_llm_dataset + tokens_per_epoch: 17841257 +num_samples: 2019 +sequence_length: 32768 +truncate_documents: false +unshuffled_epochs: 0 +unshuffled_tokens: 0 diff --git a/26_09_08_sft_email_v4_v3/dataset_cache/__mnt__queue1__shashank__exp__data__26_09_08_sft_email_v4__fastllm__shard_0_0.fast_llm_dataset_ns_4417_sl_32768_s_785266_shuffled_cumsum.npy b/26_09_08_sft_email_v4_v3/dataset_cache/__mnt__queue1__shashank__exp__data__26_09_08_sft_email_v4__fastllm__shard_0_0.fast_llm_dataset_ns_4417_sl_32768_s_785266_shuffled_cumsum.npy new file mode 100644 index 0000000000000000000000000000000000000000..30b5caddb3043036b1c186e95ea31254bd4f26ec --- /dev/null +++ b/26_09_08_sft_email_v4_v3/dataset_cache/__mnt__queue1__shashank__exp__data__26_09_08_sft_email_v4__fastllm__shard_0_0.fast_llm_dataset_ns_4417_sl_32768_s_785266_shuffled_cumsum.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b9ad75b4d0f5efc77c639274e4a7b412579961751e10483af37177e20337117 +size 17920 diff --git a/26_09_08_sft_email_v4_v3/dataset_cache/__mnt__queue1__shashank__exp__data__26_09_08_sft_email_v4__fastllm__shard_0_0.fast_llm_dataset_ns_4417_sl_32768_s_785266_shuffling.npy b/26_09_08_sft_email_v4_v3/dataset_cache/__mnt__queue1__shashank__exp__data__26_09_08_sft_email_v4__fastllm__shard_0_0.fast_llm_dataset_ns_4417_sl_32768_s_785266_shuffling.npy new file mode 100644 index 0000000000000000000000000000000000000000..b3d46de1d9ea4195f5be067cfa184fafbb8b3b2c --- /dev/null +++ b/26_09_08_sft_email_v4_v3/dataset_cache/__mnt__queue1__shashank__exp__data__26_09_08_sft_email_v4__fastllm__shard_0_0.fast_llm_dataset_ns_4417_sl_32768_s_785266_shuffling.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e96c8ba14574954551a37b3f1dd22b01ea54109124dc2369201e70ec7393a642 +size 44628 diff --git a/26_09_08_sft_email_v4_v3/dataset_cache/__mnt__queue1__shashank__exp__data__26_09_08_sft_email_v4__fastllm__shard_0_0.yaml b/26_09_08_sft_email_v4_v3/dataset_cache/__mnt__queue1__shashank__exp__data__26_09_08_sft_email_v4__fastllm__shard_0_0.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d36176d3b18cefd7b8f6c669e7a69befbc4d8d8 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/dataset_cache/__mnt__queue1__shashank__exp__data__26_09_08_sft_email_v4__fastllm__shard_0_0.yaml @@ -0,0 +1,20 @@ +config: + dataset_name: training + gpu: true + maximum_document_length: 32768 + micro_batch_size: 32768 + predicted_tokens: 1 + shuffle: epoch + token_cumsum_rate: 10 + truncate_documents: false + type: null + world_size: 32 +dataset: + documents_per_epoch: 7626 + name: __mnt__queue1__shashank__exp__data__26_09_08_sft_email_v4__fastllm__shard_0_0.fast_llm_dataset + tokens_per_epoch: 43465496 +num_samples: 4417 +sequence_length: 32768 +truncate_documents: false +unshuffled_epochs: 0 +unshuffled_tokens: 0 diff --git a/26_09_08_sft_email_v4_v3/dataset_cache/__mnt__queue1__shruthan__data__gemma_sft_replay__10k__fastllm__shard_0_0.fast_llm_dataset_ns_7008_sl_32768_s_785963_shuffled_cumsum.npy b/26_09_08_sft_email_v4_v3/dataset_cache/__mnt__queue1__shruthan__data__gemma_sft_replay__10k__fastllm__shard_0_0.fast_llm_dataset_ns_7008_sl_32768_s_785963_shuffled_cumsum.npy new file mode 100644 index 0000000000000000000000000000000000000000..ff5223adfe1cdeffbce99c957414d65cca6a4250 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/dataset_cache/__mnt__queue1__shruthan__data__gemma_sft_replay__10k__fastllm__shard_0_0.fast_llm_dataset_ns_7008_sl_32768_s_785963_shuffled_cumsum.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a682d417f335f5a4d4337a84cd615ec9622ed55d00c80b7930e2454209420cf +size 22688 diff --git a/26_09_08_sft_email_v4_v3/dataset_cache/__mnt__queue1__shruthan__data__gemma_sft_replay__10k__fastllm__shard_0_0.fast_llm_dataset_ns_7008_sl_32768_s_785963_shuffling.npy b/26_09_08_sft_email_v4_v3/dataset_cache/__mnt__queue1__shruthan__data__gemma_sft_replay__10k__fastllm__shard_0_0.fast_llm_dataset_ns_7008_sl_32768_s_785963_shuffling.npy new file mode 100644 index 0000000000000000000000000000000000000000..3e3d9d4c547eb2150427154d3f81b2bc34213f79 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/dataset_cache/__mnt__queue1__shruthan__data__gemma_sft_replay__10k__fastllm__shard_0_0.fast_llm_dataset_ns_7008_sl_32768_s_785963_shuffling.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a52f9c807bdc37b3c62c6b8ffcb6a56db72b332f03027dfd48fed767a441c39 +size 56548 diff --git a/26_09_08_sft_email_v4_v3/dataset_cache/__mnt__queue1__shruthan__data__gemma_sft_replay__10k__fastllm__shard_0_0.yaml b/26_09_08_sft_email_v4_v3/dataset_cache/__mnt__queue1__shruthan__data__gemma_sft_replay__10k__fastllm__shard_0_0.yaml new file mode 100644 index 0000000000000000000000000000000000000000..05aa3bf7526ed35deabbc50dec899f5b19345e70 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/dataset_cache/__mnt__queue1__shruthan__data__gemma_sft_replay__10k__fastllm__shard_0_0.yaml @@ -0,0 +1,20 @@ +config: + dataset_name: training + gpu: true + maximum_document_length: 32768 + micro_batch_size: 32768 + predicted_tokens: 1 + shuffle: epoch + token_cumsum_rate: 10 + truncate_documents: false + type: null + world_size: 32 +dataset: + documents_per_epoch: 10000 + name: __mnt__queue1__shruthan__data__gemma_sft_replay__10k__fastllm__shard_0_0.fast_llm_dataset + tokens_per_epoch: 67917533 +num_samples: 7008 +sequence_length: 32768 +truncate_documents: false +unshuffled_epochs: 0 +unshuffled_tokens: 0 diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/140/chat_template.jinja b/26_09_08_sft_email_v4_v3/export/gemma4/140/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..20e0028946060065b95e9376890a07f52b3c7b6f --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/140/chat_template.jinja @@ -0,0 +1,390 @@ +{# + Template: Google Gemma 4 Canonical Chat Template + Author: Google Gemma Engineering Team + Published: 2026-07-09 + Context: Fixed tool-calling loops, turn closures, and thinking content-ordering. +#} +{%- macro format_parameters(properties, required, filter_keys=false) -%} + {%- set standard_keys = ['properties', 'required', 'nullable'] -%} + {%- set ns = namespace(found_first=false) -%} + {%- for key, value in properties | dictsort -%} + {%- set add_comma = false -%} + {%- if not filter_keys or key not in standard_keys -%} + {%- if ns.found_first %},{% endif -%} + {%- set ns.found_first = true -%} + {{ key }}:{ + {%- if value['description'] -%} + description:<|"|>{{ value['description'] }}<|"|> + {%- set add_comma = true -%} + {%- endif -%} + {%- if value['type'] | upper == 'STRING' -%} + {%- if value['enum'] -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + enum:{{ format_argument(value['enum']) }} + {%- endif -%} + {%- elif value['type'] | upper == 'ARRAY' -%} + {%- if value['items'] is mapping and value['items'] -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + items:{ + {%- set ns_items = namespace(found_first=false) -%} + {%- for item_key, item_value in value['items'] | dictsort -%} + {%- if item_value is not none -%} + {%- if ns_items.found_first %},{% endif -%} + {%- set ns_items.found_first = true -%} + {%- if item_key == 'properties' -%} + properties:{ + {%- if item_value is mapping -%} + {{- format_parameters(item_value, value['items']['required'] | default([])) -}} + {%- endif -%} + } + {%- elif item_key == 'required' -%} + required:[ + {%- for req_item in item_value -%} + <|"|>{{- req_item -}}<|"|> + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + ] + {%- elif item_key == 'type' -%} + {%- if item_value is string -%} + type:{{ format_argument(item_value | upper) }} + {%- else -%} + type:{{ format_argument(item_value | map('upper') | list) }} + {%- endif -%} + {%- else -%} + {{ item_key }}:{{ format_argument(item_value) }} + {%- endif -%} + {%- endif -%} + {%- endfor -%} + } + {%- endif -%} + {%- endif -%} + {%- if value['nullable'] %} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + nullable:true + {%- endif -%} + {%- if value['type'] | upper == 'OBJECT' -%} + {%- if value['properties'] is defined and value['properties'] is mapping -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + properties:{ + {{- format_parameters(value['properties'], value['required'] | default([])) -}} + } + {%- elif value is mapping -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + properties:{ + {{- format_parameters(value, value['required'] | default([]), filter_keys=true) -}} + } + {%- endif -%} + {%- if value['required'] -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + required:[ + {%- for item in value['required'] | default([]) -%} + <|"|>{{- item -}}<|"|> + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + ] + {%- endif -%} + {%- endif -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + type:<|"|>{{ value['type'] | upper }}<|"|>} + {%- endif -%} + {%- endfor -%} +{%- endmacro -%} +{%- macro format_function_declaration(tool_data) -%} + declaration:{{- tool_data['function']['name'] -}}{description:<|"|>{{- tool_data['function']['description'] -}}<|"|> + {%- set params = tool_data['function']['parameters'] -%} + {%- if params -%} + ,parameters:{ + {%- if params['properties'] -%} + properties:{ {{- format_parameters(params['properties'], params['required']) -}} }, + {%- endif -%} + {%- if params['required'] -%} + required:[ + {%- for item in params['required'] -%} + <|"|>{{- item -}}<|"|> + {{- ',' if not loop.last -}} + {%- endfor -%} + ], + {%- endif -%} + {%- if params['type'] -%} + type:<|"|>{{- params['type'] | upper -}}<|"|>} + {%- endif -%} + {%- endif -%} + {%- if 'response' in tool_data['function'] -%} + {%- set response_declaration = tool_data['function']['response'] -%} + ,response:{ + {%- if response_declaration['description'] -%} + description:<|"|>{{- response_declaration['description'] -}}<|"|>, + {%- endif -%} + {%- if response_declaration['type'] | upper == 'OBJECT' -%} + type:<|"|>{{- response_declaration['type'] | upper -}}<|"|>} + {%- endif -%} + {%- endif -%} + } +{%- endmacro -%} +{%- macro format_argument(argument, escape_keys=True) -%} + {%- if argument is none -%} + {{- 'null' -}} + {%- elif argument is string -%} + {{- '<|"|>' + argument + '<|"|>' -}} + {%- elif argument is boolean -%} + {{- 'true' if argument else 'false' -}} + {%- elif argument is mapping -%} + {{- '{' -}} + {%- set ns = namespace(found_first=false) -%} + {%- for key, value in argument | dictsort -%} + {%- if ns.found_first %},{% endif -%} + {%- set ns.found_first = true -%} + {%- if escape_keys -%} + {{- '<|"|>' + key + '<|"|>' -}} + {%- else -%} + {{- key -}} + {%- endif -%} + :{{- format_argument(value, escape_keys=escape_keys) -}} + {%- endfor -%} + {{- '}' -}} + {%- elif argument is sequence -%} + {{- '[' -}} + {%- for item in argument -%} + {{- format_argument(item, escape_keys=escape_keys) -}} + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + {{- ']' -}} + {%- else -%} + {{- argument -}} + {%- endif -%} +{%- endmacro -%} +{%- macro strip_thinking(text) -%} + {%- set ns = namespace(result='') -%} + {%- for part in text.split('') -%} + {%- if '<|channel>' in part -%} + {%- set ns.result = ns.result + part.split('<|channel>')[0] -%} + {%- else -%} + {%- set ns.result = ns.result + part -%} + {%- endif -%} + {%- endfor -%} + {{- ns.result | trim -}} +{%- endmacro -%} + +{%- macro format_tool_response_block(tool_name, response) -%} + {{- '<|tool_response>' -}} + {%- if response is mapping -%} + {{- 'response:' + tool_name + '{' -}} + {%- for key, value in response | dictsort -%} + {{- key -}}:{{- format_argument(value, escape_keys=False) -}} + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + {{- '}' -}} + {%- else -%} + {{- 'response:' + tool_name + '{value:' + format_argument(response, escape_keys=False) + '}' -}} + {%- endif -%} + {{- '' -}} +{%- endmacro -%} + +{#- ===== SETUP ===== -#} +{%- set ns = namespace(prev_message_type=None, prev_non_tool_role=None) -%} +{%- set loop_messages = messages -%} +{%- set enable_thinking = enable_thinking | default(false) -%} +{%- set preserve_thinking = preserve_thinking | default(false) -%} +{{- bos_token -}} +{#- Handle System/Tool Definitions Block -#} +{%- if enable_thinking or tools or (messages and messages[0]['role'] in ['system', 'developer']) -%} + {{- '<|turn>system\n' -}} + {#- Inject Thinking token at the very top of the FIRST system turn -#} + {%- if enable_thinking -%} + {{- '<|think|>\n' -}} + {%- set ns.prev_message_type = 'think' -%} + {%- endif -%} + {%- if messages and messages[0]['role'] in ['system', 'developer'] -%} + {%- if messages[0]['content'] is string -%} + {{- messages[0]['content'] | trim -}} + {%- elif messages[0]['content'] is sequence -%} + {%- for item in messages[0]['content'] -%} + {{- item['text'] | trim + ' '-}} + {%- endfor -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} + {%- endif -%} + {%- if tools -%} + {%- for tool in tools %} + {{- '<|tool>' -}} + {{- format_function_declaration(tool) | trim -}} + {{- '' -}} + {%- endfor %} + {%- set ns.prev_message_type = 'tool' -%} + {%- endif -%} + {{- '\n' -}} +{%- endif %} + +{#- Pre-scan: find last user message index for reasoning guard -#} +{%- set ns_turn = namespace(last_user_idx=-1) -%} +{%- for i in range(loop_messages | length) -%} + {%- if loop_messages[i]['role'] == 'user' -%} + {%- set ns_turn.last_user_idx = i -%} + {%- endif -%} +{%- endfor -%} + +{#- Loop through messages -#} +{%- for message in loop_messages -%} + {%- if message['role'] != 'tool' -%} + {%- set ns.prev_message_type = None -%} + {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%} + {#- Detect continuation using tracked state — O(1) instead of O(n) backward scan -#} + {%- set continue_same_model_turn = (role == 'model' and ns.prev_non_tool_role == 'assistant') -%} + {%- if not continue_same_model_turn -%} + {{- '<|turn>' + role + '\n' }} + + {%- endif -%} + + {#- Render reasoning/reasoning_content as thinking channel -#} + {%- set thinking_text = message.get('reasoning') or message.get('reasoning_content') -%} + {%- set thinking_gate = (loop.index0 > ns_turn.last_user_idx) or (preserve_thinking and message.get('tool_calls')) -%} + {%- if thinking_text and thinking_gate -%} + {{- '<|channel>thought\n' + thinking_text + '\n' -}} + {%- endif -%} + + {%- if message.get('tool_calls') -%} + {%- for tool_call in message.get('tool_calls') -%} + {%- set function = tool_call['function'] -%} + {{- '<|tool_call>call:' + function['name'] + '{' -}} + {%- if function['arguments'] is mapping -%} + {%- set ns_args = namespace(found_first=false) -%} + {%- for key, value in function['arguments'] | dictsort -%} + {%- if ns_args.found_first %},{% endif -%} + {%- set ns_args.found_first = true -%} + {{- key -}}:{{- format_argument(value, escape_keys=False) -}} + {%- endfor -%} + {%- elif function['arguments'] is none -%} + {%- else -%} + {{- raise_exception( + "chat_template: tool_calls[].function.arguments must be a " + "JSON object (mapping), not a string. Deserialize arguments " + "before passing to the template." + ) -}} + {%- endif -%} + {{- '}' -}} + {%- endfor -%} + {%- set ns.prev_message_type = 'tool_call' -%} + {%- endif -%} + + {%- set ns_tr_out = namespace(flag=false) -%} + {%- if message.get('tool_responses') -%} + {#- Legacy: tool_responses embedded on the assistant message (Google/Gemma native) -#} + {%- for tool_response in message.get('tool_responses') -%} + {{- format_tool_response_block(tool_response['name'] | default('unknown', true), tool_response['response']) -}} + {%- set ns_tr_out.flag = true -%} + {%- set ns.prev_message_type = 'tool_response' -%} + {%- endfor -%} + {%- elif message.get('tool_calls') -%} + {#- OpenAI Chat Completions: forward-scan consecutive role:tool messages -#} + {%- set ns_tool_scan = namespace(stopped=false) -%} + {%- for k in range(loop.index0 + 1, loop_messages | length) -%} + {%- if ns_tool_scan.stopped -%} + {%- elif loop_messages[k]['role'] != 'tool' -%} + {%- set ns_tool_scan.stopped = true -%} + {%- else -%} + {%- set follow = loop_messages[k] -%} + {#- Resolve tool_call_id to function name -#} + {%- set ns_tname = namespace(name=follow.get('name') or 'unknown') -%} + {%- for tc in message.get('tool_calls') -%} + {%- if tc.get('id') == follow.get('tool_call_id') -%} + {%- set ns_tname.name = tc['function']['name'] -%} + {%- endif -%} + {%- endfor -%} + {#- Handle content as string or content-parts array -#} + {%- set tool_body = follow.get('content') -%} + {%- if tool_body is string -%} + {{- format_tool_response_block(ns_tname.name, tool_body) -}} + {%- elif tool_body is sequence and tool_body is not string -%} + {%- set ns_txt = namespace(s='') -%} + {%- for part in tool_body -%} + {%- if part.get('type') == 'text' -%} + {%- set ns_txt.s = ns_txt.s + (part.get('text') | default('')) -%} + {%- endif -%} + {%- endfor -%} + {{- format_tool_response_block(ns_tname.name, ns_txt.s) -}} + {%- for part in tool_body -%} + {%- if part.get('type') in ['image', 'image_url'] -%} + {{- '<|image|>' -}} + {%- elif part.get('type') in ['audio', 'input_audio'] -%} + {{- '<|audio|>' -}} + {%- elif part.get('type') == 'video' -%} + {{- '<|video|>' -}} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{- format_tool_response_block(ns_tname.name, tool_body) -}} + {%- endif -%} + {%- set ns_tr_out.flag = true -%} + {%- set ns.prev_message_type = 'tool_response' -%} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + + {%- set captured_content -%} + {%- if message.get('content') is string -%} + {%- if role == 'model' -%} + {{- strip_thinking(message['content']) -}} + {%- else -%} + {{- message['content'] | trim -}} + {%- endif -%} + {%- elif message.get('content') is sequence -%} + {%- for item in message['content'] -%} + {%- if item.get('type') == 'text' -%} + {%- if role == 'model' -%} + {{- strip_thinking(item['text']) -}} + {%- else -%} + {{- item['text'] | trim -}} + {%- endif -%} + {%- elif item.get('type') in ['image', 'image_url'] -%} + {{- '<|image|>' -}} + {%- elif item.get('type') in ['audio', 'input_audio'] -%} + {{- '<|audio|>' -}} + {%- elif item.get('type') == 'video' -%} + {{- '<|video|>' -}} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + {%- endset -%} + + {{- captured_content -}} + {%- set has_content = captured_content | trim | length > 0 -%} + + {#- Forward-scan: find next non-tool message role for continuation detection -#} + {%- set next_nt = namespace(role=None, found=false) -%} + {%- for j in range(loop.index0 + 1, loop_messages | length) -%} + {%- if not next_nt.found -%} + {%- if loop_messages[j]['role'] != 'tool' -%} + {%- set next_nt.role = loop_messages[j]['role'] -%} + {%- set next_nt.found = true -%} + {%- endif -%} + {%- endif -%} + {%- endfor -%} + + {%- set continues_into_next = ( + role == 'model' + and next_nt.role == 'assistant' + and (not message.get('tool_calls') or ns_tr_out.flag) + ) -%} + + {%- if ns.prev_message_type == 'tool_call' and not ns_tr_out.flag -%} + {{- '<|tool_response>' -}} + {%- elif continues_into_next -%} + {%- elif not (ns_tr_out.flag and not has_content and not next_nt.found) -%} + {{- '\n' -}} + {%- endif -%} + + {#- Track previous non-tool role for next iteration (avoids O(n) backward scan) -#} + {%- set ns.prev_non_tool_role = message['role'] -%} + {%- endif -%} +{%- endfor -%} + +{%- if add_generation_prompt -%} + {%- if ns.prev_message_type != 'tool_response' and ns.prev_message_type != 'tool_call' -%} + {{- '<|turn>model\n' -}} + {%- if not enable_thinking -%} + {{- '<|channel>thought\n' -}} + {%- endif -%} + {%- elif ns.prev_message_type == 'tool_response' and enable_thinking -%} + {{- '<|channel>thought\n' -}} + {%- endif -%} +{%- endif -%} diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/140/config.json b/26_09_08_sft_email_v4_v3/export/gemma4/140/config.json new file mode 100644 index 0000000000000000000000000000000000000000..518607d204ad0bf72d56220356c72e1516d05041 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/140/config.json @@ -0,0 +1,83 @@ +{ + "attention_bias": false, + "attention_dropout": 0.0, + "attention_k_eq_v": true, + "bos_token_id": 2, + "dtype": "bfloat16", + "enable_moe_block": true, + "eos_token_id": 1, + "final_logit_softcapping": 30.0, + "global_head_dim": 512, + "head_dim": 256, + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 2816, + "hidden_size_per_layer_input": 0, + "initializer_range": 0.02, + "intermediate_size": 2112, + "layer_types": [ + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 262144, + "model_type": "gemma4_text", + "moe_intermediate_size": 704, + "num_attention_heads": 16, + "num_experts": 128, + "num_global_key_value_heads": 2, + "num_hidden_layers": 30, + "num_key_value_heads": 8, + "num_kv_shared_layers": 0, + "pad_token_id": 0, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "full_attention": { + "partial_rotary_factor": 0.25, + "rope_theta": 1000000.0, + "rope_type": "proportional" + }, + "sliding_attention": { + "rope_theta": 10000.0, + "rope_type": "default" + } + }, + "sliding_window": 1024, + "tie_word_embeddings": true, + "top_k_experts": 8, + "use_bidirectional_attention": "vision", + "use_cache": true, + "use_double_wide_mlp": false, + "vocab_size": 262144, + "vocab_size_per_layer_input": 262144, + "architectures": [ + "Gemma4ForCausalLM" + ], + "transformers_version": "5.5.0.dev0" +} \ No newline at end of file diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/140/generation_config.json b/26_09_08_sft_email_v4_v3/export/gemma4/140/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e605bb4523b1462ea9d9a3810b9e3ecf7ab7b1f6 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/140/generation_config.json @@ -0,0 +1,14 @@ +{ + "bos_token_id": 2, + "do_sample": true, + "eos_token_id": [ + 1, + 106, + 50 + ], + "pad_token_id": 0, + "temperature": 1.0, + "top_k": 64, + "top_p": 0.95, + "transformers_version": "5.5.0.dev0" +} diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/140/model.safetensors.index.json b/26_09_08_sft_email_v4_v3/export/gemma4/140/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..3db38910ebe05f719978c7acd4c4e4f00d2adc44 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/140/model.safetensors.index.json @@ -0,0 +1,1053 @@ +{ + "metadata": { + "fast_llm_metadata": { + "fast_llm_version": "0.4.0", + "model": "gpt", + "format": "gemma4", + "config": { + "type": "gpt", + "base_model": { + "decoder": { + "type": "pattern", + "blocks": { + "full_attention": { + "type": "decoder", + "mixer": { + "type": "attention", + "rotary": { + "type": "proportional", + "theta": 1000000.0, + "partial_rotary_factor": 0.25 + }, + "heads": 16, + "head_groups": 2, + "head_size": 512, + "add_linear_biases": false, + "dropout": 0.0, + "softmax_scale_power": 0.0, + "implementation": "sdpa", + "query_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "key_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "value_norm": { + "type": "fixed_rms_norm", + "epsilon": 1e-06 + }, + "shared_key_value": true + }, + "mlp": { + "type": "hybrid_moe", + "dense": { + "type": "mlp", + "pre_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "post_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "intermediate_size": 2112, + "add_linear_biases": false, + "gated": true, + "activation": "gelu", + "recompute_level": "full" + }, + "routed": { + "type": "moe", + "pre_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "post_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "intermediate_size": 704, + "add_linear_biases": false, + "gated": true, + "activation": "gelu", + "recompute_level": "full", + "router_normalization": { + "type": "fixed_rms_norm", + "epsilon": 1e-06 + }, + "router_scale": { + "enabled": true + }, + "router_input_scale": 0.018844459036110227, + "router_per_expert_scale": { + "enabled": true + }, + "experts": 128, + "experts_per_token": 8, + "auxiliary_loss_coefficient": 0.0 + } + }, + "normalization": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "pre_mlp_normalization": { + "type": "none" + }, + "post_mixer_normalization": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "post_mlp_normalization": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "output_scale": { + "lr_scale": 0.0, + "enabled": true + } + }, + "sliding_attention": { + "type": "decoder", + "mixer": { + "type": "attention", + "rotary": { + "type": "default", + "theta": 10000.0 + }, + "heads": 16, + "head_groups": 8, + "head_size": 256, + "add_linear_biases": false, + "dropout": 0.0, + "window_size": 1024, + "softmax_scale_power": 0.0, + "implementation": "flash", + "query_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "key_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "value_norm": { + "type": "fixed_rms_norm", + "epsilon": 1e-06 + } + }, + "mlp": { + "type": "hybrid_moe", + "dense": { + "type": "mlp", + "pre_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "post_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "intermediate_size": 2112, + "add_linear_biases": false, + "gated": true, + "activation": "gelu", + "recompute_level": "full" + }, + "routed": { + "type": "moe", + "pre_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "post_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "intermediate_size": 704, + "add_linear_biases": false, + "gated": true, + "activation": "gelu", + "recompute_level": "full", + "router_normalization": { + "type": "fixed_rms_norm", + "epsilon": 1e-06 + }, + "router_scale": { + "enabled": true + }, + "router_input_scale": 0.018844459036110227, + "router_per_expert_scale": { + "enabled": true + }, + "experts": 128, + "experts_per_token": 8, + "auxiliary_loss_coefficient": 0.0 + } + }, + "normalization": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "pre_mlp_normalization": { + "type": "none" + }, + "post_mixer_normalization": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "post_mlp_normalization": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "output_scale": { + "lr_scale": 0.0, + "enabled": true + } + } + }, + "pattern": [ + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention" + ], + "num_blocks": 30 + }, + "embeddings": { + "vocab_size": 262144, + "vocab_parallel": false, + "embedding_scale": 53.0659966456864 + }, + "head": { + "normalization": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "cross_entropy_splits": 8, + "final_logit_softcap": 30.0 + }, + "tied_embedding_weight": true, + "hidden_size": 2816, + "peft": { + "type": "none" + } + }, + "multi_stage": { + "zero_stage": 3 + }, + "distributed": { + "tensor_parallel": 2, + "sequence_tensor_parallel": true, + "sequence_data_parallel": 4, + "world_size": 32, + "rank": 0, + "local_world_size": 8, + "timeout": 3600.0, + "compute_dtype": "bfloat16" + } + }, + "shards": [ + "weights" + ], + "metadata": { + "optimizer": { + "current_step": 140, + "grad_scaler": { + "type": "NoopGradScaler" + } + }, + "completed_steps": 140, + "metrics": { + "training": { + "batch_size": 2097152, + "load_balancing_loss": 828.73388671875, + "lm_head_loss": 0.11952575296163559, + "total_steps": 210, + "completed_steps": 140, + "consumed_tokens": 293601280, + "percent_done": 66.66666666666667, + "step_time_ms": 26989.178344607353, + "step_time_average_ms": 30017.633229094958, + "remaining_time": 2101.234326036647, + "completion_time": 1788873030.2734404, + "skipped_iters": 0, + "nan_iters": 0, + "model_tflops": 62205.328686694214, + "hardware_tflops": 77174.52198293927, + "tokens_per_sec_per_gpu": 2428.232499826902, + "run": 0, + "grad_norm": 2.5687570571899414, + "learning_rate": 5.3159155930020984e-06, + "loss_scale": 1.0, + "reserved": 55562.0, + "allocated": 21478.49755859375, + "max_allocated": 54799.8583984375, + "max_reserved": 55562.0, + "global_max_reserved": 74782.0 + } + } + } + }, + "model_config": { + "vocab_size": 262144, + "num_attention_heads": 16, + "num_key_value_heads": 8, + "head_dim": 256, + "global_head_dim": 512, + "num_global_key_value_heads": 2, + "attention_bias": false, + "attention_dropout": 0.0, + "sliding_window": 1024, + "rms_norm_eps": 1e-06, + "attention_k_eq_v": true, + "rope_parameters": { + "sliding_attention": { + "rope_type": "default", + "rope_theta": 10000.0 + }, + "full_attention": { + "rope_type": "proportional", + "rope_theta": 1000000.0, + "partial_rotary_factor": 0.25 + } + }, + "intermediate_size": 2112, + "hidden_activation": "gelu_pytorch_tanh", + "num_experts": 128, + "top_k_experts": 8, + "moe_intermediate_size": 704, + "enable_moe_block": true, + "num_hidden_layers": 30, + "layer_types": [ + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention" + ], + "final_logit_softcapping": 30.0, + "tie_word_embeddings": true, + "hidden_size": 2816, + "hidden_size_per_layer_input": 0, + "use_bidirectional_attention": null, + "model_type": "gemma4_text", + "architectures": [ + "Gemma4ForCausalLM" + ] + }, + "format": "pt" + }, + "weight_map": { + "model.embed_tokens.weight": "model_0.safetensors", + "model.layers.0.input_layernorm.weight": "model_0.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model_0.safetensors", + "model.layers.0.post_feedforward_layernorm.weight": "model_0.safetensors", + "model.layers.0.self_attn.q_norm.weight": "model_0.safetensors", + "model.layers.0.self_attn.k_norm.weight": "model_0.safetensors", + "model.layers.0.pre_feedforward_layernorm.weight": "model_0.safetensors", + "model.layers.0.post_feedforward_layernorm_1.weight": "model_0.safetensors", + "model.layers.0.router.scale": "model_0.safetensors", + "model.layers.0.router.per_expert_scale": "model_0.safetensors", + "model.layers.0.pre_feedforward_layernorm_2.weight": "model_0.safetensors", + "model.layers.0.post_feedforward_layernorm_2.weight": "model_0.safetensors", + "model.layers.0.router.proj.weight": "model_0.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model_0.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model_0.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model_0.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model_0.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model_0.safetensors", + "model.layers.0.mlp.up_proj.weight": "model_0.safetensors", + "model.layers.0.mlp.down_proj.weight": "model_0.safetensors", + "model.layers.0.experts.gate_up_proj": "model_0.safetensors", + "model.layers.0.experts.down_proj": "model_0.safetensors", + "model.layers.0.layer_scalar": "model_0.safetensors", + "model.layers.1.input_layernorm.weight": "model_0.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model_0.safetensors", + "model.layers.1.post_feedforward_layernorm.weight": "model_0.safetensors", + "model.layers.1.self_attn.q_norm.weight": "model_0.safetensors", + "model.layers.1.self_attn.k_norm.weight": "model_0.safetensors", + "model.layers.1.pre_feedforward_layernorm.weight": "model_0.safetensors", + "model.layers.1.post_feedforward_layernorm_1.weight": "model_0.safetensors", + "model.layers.1.router.scale": "model_0.safetensors", + "model.layers.1.router.per_expert_scale": "model_0.safetensors", + "model.layers.1.pre_feedforward_layernorm_2.weight": "model_0.safetensors", + "model.layers.1.post_feedforward_layernorm_2.weight": "model_0.safetensors", + "model.layers.1.router.proj.weight": "model_0.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model_0.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model_0.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model_0.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model_0.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model_0.safetensors", + "model.layers.1.mlp.up_proj.weight": "model_0.safetensors", + "model.layers.1.mlp.down_proj.weight": "model_0.safetensors", + "model.layers.1.experts.gate_up_proj": "model_0.safetensors", + "model.layers.1.experts.down_proj": "model_0.safetensors", + "model.layers.1.layer_scalar": "model_0.safetensors", + "model.layers.2.input_layernorm.weight": "model_0.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model_0.safetensors", + "model.layers.2.post_feedforward_layernorm.weight": "model_0.safetensors", + "model.layers.2.self_attn.q_norm.weight": "model_0.safetensors", + "model.layers.2.self_attn.k_norm.weight": "model_0.safetensors", + "model.layers.2.pre_feedforward_layernorm.weight": "model_0.safetensors", + "model.layers.2.post_feedforward_layernorm_1.weight": "model_0.safetensors", + "model.layers.2.router.scale": "model_0.safetensors", + "model.layers.2.router.per_expert_scale": "model_0.safetensors", + "model.layers.2.pre_feedforward_layernorm_2.weight": "model_0.safetensors", + "model.layers.2.post_feedforward_layernorm_2.weight": "model_0.safetensors", + "model.layers.2.router.proj.weight": "model_0.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model_0.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model_0.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model_0.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model_0.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model_0.safetensors", + "model.layers.2.mlp.up_proj.weight": "model_0.safetensors", + "model.layers.2.mlp.down_proj.weight": "model_0.safetensors", + "model.layers.2.experts.gate_up_proj": "model_0.safetensors", + "model.layers.2.experts.down_proj": "model_0.safetensors", + "model.layers.2.layer_scalar": "model_0.safetensors", + "model.layers.3.input_layernorm.weight": "model_0.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model_0.safetensors", + "model.layers.3.post_feedforward_layernorm.weight": "model_0.safetensors", + "model.layers.3.self_attn.q_norm.weight": "model_0.safetensors", + "model.layers.3.self_attn.k_norm.weight": "model_0.safetensors", + "model.layers.3.pre_feedforward_layernorm.weight": "model_0.safetensors", + "model.layers.3.post_feedforward_layernorm_1.weight": "model_0.safetensors", + "model.layers.3.router.scale": "model_0.safetensors", + "model.layers.3.router.per_expert_scale": "model_0.safetensors", + "model.layers.3.pre_feedforward_layernorm_2.weight": "model_0.safetensors", + "model.layers.3.post_feedforward_layernorm_2.weight": "model_0.safetensors", + "model.layers.3.router.proj.weight": "model_0.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model_0.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model_0.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model_0.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model_0.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model_0.safetensors", + "model.layers.3.mlp.up_proj.weight": "model_0.safetensors", + "model.layers.3.mlp.down_proj.weight": "model_0.safetensors", + "model.layers.3.experts.gate_up_proj": "model_0.safetensors", + "model.layers.3.experts.down_proj": "model_0.safetensors", + "model.layers.3.layer_scalar": "model_0.safetensors", + "model.layers.4.input_layernorm.weight": "model_0.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model_0.safetensors", + "model.layers.4.post_feedforward_layernorm.weight": "model_0.safetensors", + "model.layers.4.self_attn.q_norm.weight": "model_0.safetensors", + "model.layers.4.self_attn.k_norm.weight": "model_0.safetensors", + "model.layers.4.pre_feedforward_layernorm.weight": "model_0.safetensors", + "model.layers.4.post_feedforward_layernorm_1.weight": "model_0.safetensors", + "model.layers.4.router.scale": "model_0.safetensors", + "model.layers.4.router.per_expert_scale": "model_0.safetensors", + "model.layers.4.pre_feedforward_layernorm_2.weight": "model_0.safetensors", + "model.layers.4.post_feedforward_layernorm_2.weight": "model_0.safetensors", + "model.layers.4.router.proj.weight": "model_0.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model_0.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model_0.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model_0.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model_0.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model_0.safetensors", + "model.layers.4.mlp.up_proj.weight": "model_0.safetensors", + "model.layers.4.mlp.down_proj.weight": "model_0.safetensors", + "model.layers.4.experts.gate_up_proj": "model_0.safetensors", + "model.layers.4.experts.down_proj": "model_1.safetensors", + "model.layers.4.layer_scalar": "model_1.safetensors", + "model.layers.5.input_layernorm.weight": "model_1.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model_1.safetensors", + "model.layers.5.post_feedforward_layernorm.weight": "model_1.safetensors", + "model.layers.5.self_attn.q_norm.weight": "model_1.safetensors", + "model.layers.5.self_attn.k_norm.weight": "model_1.safetensors", + "model.layers.5.pre_feedforward_layernorm.weight": "model_1.safetensors", + "model.layers.5.post_feedforward_layernorm_1.weight": "model_1.safetensors", + "model.layers.5.router.scale": "model_1.safetensors", + "model.layers.5.router.per_expert_scale": "model_1.safetensors", + "model.layers.5.pre_feedforward_layernorm_2.weight": "model_1.safetensors", + "model.layers.5.post_feedforward_layernorm_2.weight": "model_1.safetensors", + "model.layers.5.router.proj.weight": "model_1.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model_1.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model_1.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model_1.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model_1.safetensors", + "model.layers.5.mlp.up_proj.weight": "model_1.safetensors", + "model.layers.5.mlp.down_proj.weight": "model_1.safetensors", + "model.layers.5.experts.gate_up_proj": "model_1.safetensors", + "model.layers.5.experts.down_proj": "model_1.safetensors", + "model.layers.5.layer_scalar": "model_1.safetensors", + "model.layers.6.input_layernorm.weight": "model_1.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model_1.safetensors", + "model.layers.6.post_feedforward_layernorm.weight": "model_1.safetensors", + "model.layers.6.self_attn.q_norm.weight": "model_1.safetensors", + "model.layers.6.self_attn.k_norm.weight": "model_1.safetensors", + "model.layers.6.pre_feedforward_layernorm.weight": "model_1.safetensors", + "model.layers.6.post_feedforward_layernorm_1.weight": "model_1.safetensors", + "model.layers.6.router.scale": "model_1.safetensors", + "model.layers.6.router.per_expert_scale": "model_1.safetensors", + "model.layers.6.pre_feedforward_layernorm_2.weight": "model_1.safetensors", + "model.layers.6.post_feedforward_layernorm_2.weight": "model_1.safetensors", + "model.layers.6.router.proj.weight": "model_1.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model_1.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model_1.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model_1.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model_1.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model_1.safetensors", + "model.layers.6.mlp.up_proj.weight": "model_1.safetensors", + "model.layers.6.mlp.down_proj.weight": "model_1.safetensors", + "model.layers.6.experts.gate_up_proj": "model_1.safetensors", + "model.layers.6.experts.down_proj": "model_1.safetensors", + "model.layers.6.layer_scalar": "model_1.safetensors", + "model.layers.7.input_layernorm.weight": "model_1.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model_1.safetensors", + "model.layers.7.post_feedforward_layernorm.weight": "model_1.safetensors", + "model.layers.7.self_attn.q_norm.weight": "model_1.safetensors", + "model.layers.7.self_attn.k_norm.weight": "model_1.safetensors", + "model.layers.7.pre_feedforward_layernorm.weight": "model_1.safetensors", + "model.layers.7.post_feedforward_layernorm_1.weight": "model_1.safetensors", + "model.layers.7.router.scale": "model_1.safetensors", + "model.layers.7.router.per_expert_scale": "model_1.safetensors", + "model.layers.7.pre_feedforward_layernorm_2.weight": "model_1.safetensors", + "model.layers.7.post_feedforward_layernorm_2.weight": "model_1.safetensors", + "model.layers.7.router.proj.weight": "model_1.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model_1.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model_1.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model_1.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model_1.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model_1.safetensors", + "model.layers.7.mlp.up_proj.weight": "model_1.safetensors", + "model.layers.7.mlp.down_proj.weight": "model_1.safetensors", + "model.layers.7.experts.gate_up_proj": "model_1.safetensors", + "model.layers.7.experts.down_proj": "model_1.safetensors", + "model.layers.7.layer_scalar": "model_1.safetensors", + "model.layers.8.input_layernorm.weight": "model_1.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model_1.safetensors", + "model.layers.8.post_feedforward_layernorm.weight": "model_1.safetensors", + "model.layers.8.self_attn.q_norm.weight": "model_1.safetensors", + "model.layers.8.self_attn.k_norm.weight": "model_1.safetensors", + "model.layers.8.pre_feedforward_layernorm.weight": "model_1.safetensors", + "model.layers.8.post_feedforward_layernorm_1.weight": "model_1.safetensors", + "model.layers.8.router.scale": "model_1.safetensors", + "model.layers.8.router.per_expert_scale": "model_1.safetensors", + "model.layers.8.pre_feedforward_layernorm_2.weight": "model_1.safetensors", + "model.layers.8.post_feedforward_layernorm_2.weight": "model_1.safetensors", + "model.layers.8.router.proj.weight": "model_1.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model_1.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model_1.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model_1.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model_1.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model_1.safetensors", + "model.layers.8.mlp.up_proj.weight": "model_1.safetensors", + "model.layers.8.mlp.down_proj.weight": "model_1.safetensors", + "model.layers.8.experts.gate_up_proj": "model_1.safetensors", + "model.layers.8.experts.down_proj": "model_1.safetensors", + "model.layers.8.layer_scalar": "model_1.safetensors", + "model.layers.9.input_layernorm.weight": "model_1.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model_1.safetensors", + "model.layers.9.post_feedforward_layernorm.weight": "model_1.safetensors", + "model.layers.9.self_attn.q_norm.weight": "model_1.safetensors", + "model.layers.9.self_attn.k_norm.weight": "model_1.safetensors", + "model.layers.9.pre_feedforward_layernorm.weight": "model_1.safetensors", + "model.layers.9.post_feedforward_layernorm_1.weight": "model_1.safetensors", + "model.layers.9.router.scale": "model_1.safetensors", + "model.layers.9.router.per_expert_scale": "model_1.safetensors", + "model.layers.9.pre_feedforward_layernorm_2.weight": "model_1.safetensors", + "model.layers.9.post_feedforward_layernorm_2.weight": "model_1.safetensors", + "model.layers.9.router.proj.weight": "model_1.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model_1.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model_1.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model_1.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model_1.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model_1.safetensors", + "model.layers.9.mlp.up_proj.weight": "model_1.safetensors", + "model.layers.9.mlp.down_proj.weight": "model_1.safetensors", + "model.layers.9.experts.gate_up_proj": "model_1.safetensors", + "model.layers.9.experts.down_proj": "model_1.safetensors", + "model.layers.9.layer_scalar": "model_2.safetensors", + "model.layers.10.input_layernorm.weight": "model_2.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model_2.safetensors", + "model.layers.10.post_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.10.self_attn.q_norm.weight": "model_2.safetensors", + "model.layers.10.self_attn.k_norm.weight": "model_2.safetensors", + "model.layers.10.pre_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.10.post_feedforward_layernorm_1.weight": "model_2.safetensors", + "model.layers.10.router.scale": "model_2.safetensors", + "model.layers.10.router.per_expert_scale": "model_2.safetensors", + "model.layers.10.pre_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.10.post_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.10.router.proj.weight": "model_2.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model_2.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model_2.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model_2.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model_2.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model_2.safetensors", + "model.layers.10.mlp.up_proj.weight": "model_2.safetensors", + "model.layers.10.mlp.down_proj.weight": "model_2.safetensors", + "model.layers.10.experts.gate_up_proj": "model_2.safetensors", + "model.layers.10.experts.down_proj": "model_2.safetensors", + "model.layers.10.layer_scalar": "model_2.safetensors", + "model.layers.11.input_layernorm.weight": "model_2.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model_2.safetensors", + "model.layers.11.post_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.11.self_attn.q_norm.weight": "model_2.safetensors", + "model.layers.11.self_attn.k_norm.weight": "model_2.safetensors", + "model.layers.11.pre_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.11.post_feedforward_layernorm_1.weight": "model_2.safetensors", + "model.layers.11.router.scale": "model_2.safetensors", + "model.layers.11.router.per_expert_scale": "model_2.safetensors", + "model.layers.11.pre_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.11.post_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.11.router.proj.weight": "model_2.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model_2.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model_2.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model_2.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model_2.safetensors", + "model.layers.11.mlp.up_proj.weight": "model_2.safetensors", + "model.layers.11.mlp.down_proj.weight": "model_2.safetensors", + "model.layers.11.experts.gate_up_proj": "model_2.safetensors", + "model.layers.11.experts.down_proj": "model_2.safetensors", + "model.layers.11.layer_scalar": "model_2.safetensors", + "model.layers.12.input_layernorm.weight": "model_2.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model_2.safetensors", + "model.layers.12.post_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.12.self_attn.q_norm.weight": "model_2.safetensors", + "model.layers.12.self_attn.k_norm.weight": "model_2.safetensors", + "model.layers.12.pre_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.12.post_feedforward_layernorm_1.weight": "model_2.safetensors", + "model.layers.12.router.scale": "model_2.safetensors", + "model.layers.12.router.per_expert_scale": "model_2.safetensors", + "model.layers.12.pre_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.12.post_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.12.router.proj.weight": "model_2.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model_2.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model_2.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model_2.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model_2.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model_2.safetensors", + "model.layers.12.mlp.up_proj.weight": "model_2.safetensors", + "model.layers.12.mlp.down_proj.weight": "model_2.safetensors", + "model.layers.12.experts.gate_up_proj": "model_2.safetensors", + "model.layers.12.experts.down_proj": "model_2.safetensors", + "model.layers.12.layer_scalar": "model_2.safetensors", + "model.layers.13.input_layernorm.weight": "model_2.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model_2.safetensors", + "model.layers.13.post_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.13.self_attn.q_norm.weight": "model_2.safetensors", + "model.layers.13.self_attn.k_norm.weight": "model_2.safetensors", + "model.layers.13.pre_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.13.post_feedforward_layernorm_1.weight": "model_2.safetensors", + "model.layers.13.router.scale": "model_2.safetensors", + "model.layers.13.router.per_expert_scale": "model_2.safetensors", + "model.layers.13.pre_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.13.post_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.13.router.proj.weight": "model_2.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model_2.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model_2.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model_2.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model_2.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model_2.safetensors", + "model.layers.13.mlp.up_proj.weight": "model_2.safetensors", + "model.layers.13.mlp.down_proj.weight": "model_2.safetensors", + "model.layers.13.experts.gate_up_proj": "model_2.safetensors", + "model.layers.13.experts.down_proj": "model_2.safetensors", + "model.layers.13.layer_scalar": "model_2.safetensors", + "model.layers.14.input_layernorm.weight": "model_2.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model_2.safetensors", + "model.layers.14.post_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.14.self_attn.q_norm.weight": "model_2.safetensors", + "model.layers.14.self_attn.k_norm.weight": "model_2.safetensors", + "model.layers.14.pre_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.14.post_feedforward_layernorm_1.weight": "model_2.safetensors", + "model.layers.14.router.scale": "model_2.safetensors", + "model.layers.14.router.per_expert_scale": "model_2.safetensors", + "model.layers.14.pre_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.14.post_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.14.router.proj.weight": "model_2.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model_2.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model_2.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model_2.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model_2.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model_2.safetensors", + "model.layers.14.mlp.up_proj.weight": "model_2.safetensors", + "model.layers.14.mlp.down_proj.weight": "model_2.safetensors", + "model.layers.14.experts.gate_up_proj": "model_2.safetensors", + "model.layers.14.experts.down_proj": "model_2.safetensors", + "model.layers.14.layer_scalar": "model_2.safetensors", + "model.layers.15.input_layernorm.weight": "model_2.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model_2.safetensors", + "model.layers.15.post_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.15.self_attn.q_norm.weight": "model_2.safetensors", + "model.layers.15.self_attn.k_norm.weight": "model_2.safetensors", + "model.layers.15.pre_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.15.post_feedforward_layernorm_1.weight": "model_2.safetensors", + "model.layers.15.router.scale": "model_2.safetensors", + "model.layers.15.router.per_expert_scale": "model_2.safetensors", + "model.layers.15.pre_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.15.post_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.15.router.proj.weight": "model_2.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model_2.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model_2.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model_2.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model_2.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model_2.safetensors", + "model.layers.15.mlp.up_proj.weight": "model_2.safetensors", + "model.layers.15.mlp.down_proj.weight": "model_2.safetensors", + "model.layers.15.experts.gate_up_proj": "model_2.safetensors", + "model.layers.15.experts.down_proj": "model_3.safetensors", + "model.layers.15.layer_scalar": "model_3.safetensors", + "model.layers.16.input_layernorm.weight": "model_3.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model_3.safetensors", + "model.layers.16.post_feedforward_layernorm.weight": "model_3.safetensors", + "model.layers.16.self_attn.q_norm.weight": "model_3.safetensors", + "model.layers.16.self_attn.k_norm.weight": "model_3.safetensors", + "model.layers.16.pre_feedforward_layernorm.weight": "model_3.safetensors", + "model.layers.16.post_feedforward_layernorm_1.weight": "model_3.safetensors", + "model.layers.16.router.scale": "model_3.safetensors", + "model.layers.16.router.per_expert_scale": "model_3.safetensors", + "model.layers.16.pre_feedforward_layernorm_2.weight": "model_3.safetensors", + "model.layers.16.post_feedforward_layernorm_2.weight": "model_3.safetensors", + "model.layers.16.router.proj.weight": "model_3.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model_3.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model_3.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model_3.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model_3.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model_3.safetensors", + "model.layers.16.mlp.up_proj.weight": "model_3.safetensors", + "model.layers.16.mlp.down_proj.weight": "model_3.safetensors", + "model.layers.16.experts.gate_up_proj": "model_3.safetensors", + "model.layers.16.experts.down_proj": "model_3.safetensors", + "model.layers.16.layer_scalar": "model_3.safetensors", + "model.layers.17.input_layernorm.weight": "model_3.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model_3.safetensors", + "model.layers.17.post_feedforward_layernorm.weight": "model_3.safetensors", + "model.layers.17.self_attn.q_norm.weight": "model_3.safetensors", + "model.layers.17.self_attn.k_norm.weight": "model_3.safetensors", + "model.layers.17.pre_feedforward_layernorm.weight": "model_3.safetensors", + "model.layers.17.post_feedforward_layernorm_1.weight": "model_3.safetensors", + "model.layers.17.router.scale": "model_3.safetensors", + "model.layers.17.router.per_expert_scale": "model_3.safetensors", + "model.layers.17.pre_feedforward_layernorm_2.weight": "model_3.safetensors", + "model.layers.17.post_feedforward_layernorm_2.weight": "model_3.safetensors", + "model.layers.17.router.proj.weight": "model_3.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model_3.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model_3.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model_3.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model_3.safetensors", + "model.layers.17.mlp.up_proj.weight": "model_3.safetensors", + "model.layers.17.mlp.down_proj.weight": "model_3.safetensors", + "model.layers.17.experts.gate_up_proj": "model_3.safetensors", + "model.layers.17.experts.down_proj": "model_3.safetensors", + "model.layers.17.layer_scalar": "model_3.safetensors", + "model.layers.18.input_layernorm.weight": "model_3.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model_3.safetensors", + "model.layers.18.post_feedforward_layernorm.weight": "model_3.safetensors", + "model.layers.18.self_attn.q_norm.weight": "model_3.safetensors", + "model.layers.18.self_attn.k_norm.weight": "model_3.safetensors", + "model.layers.18.pre_feedforward_layernorm.weight": "model_3.safetensors", + "model.layers.18.post_feedforward_layernorm_1.weight": "model_3.safetensors", + "model.layers.18.router.scale": "model_3.safetensors", + "model.layers.18.router.per_expert_scale": "model_3.safetensors", + "model.layers.18.pre_feedforward_layernorm_2.weight": "model_3.safetensors", + "model.layers.18.post_feedforward_layernorm_2.weight": "model_3.safetensors", + "model.layers.18.router.proj.weight": "model_3.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model_3.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model_3.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model_3.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model_3.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model_3.safetensors", + "model.layers.18.mlp.up_proj.weight": "model_3.safetensors", + "model.layers.18.mlp.down_proj.weight": "model_3.safetensors", + "model.layers.18.experts.gate_up_proj": "model_3.safetensors", + "model.layers.18.experts.down_proj": "model_3.safetensors", + "model.layers.18.layer_scalar": "model_3.safetensors", + "model.layers.19.input_layernorm.weight": "model_3.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model_3.safetensors", + "model.layers.19.post_feedforward_layernorm.weight": "model_3.safetensors", + "model.layers.19.self_attn.q_norm.weight": "model_3.safetensors", + "model.layers.19.self_attn.k_norm.weight": "model_3.safetensors", + "model.layers.19.pre_feedforward_layernorm.weight": "model_3.safetensors", + "model.layers.19.post_feedforward_layernorm_1.weight": "model_3.safetensors", + "model.layers.19.router.scale": "model_3.safetensors", + "model.layers.19.router.per_expert_scale": "model_3.safetensors", + "model.layers.19.pre_feedforward_layernorm_2.weight": "model_3.safetensors", + "model.layers.19.post_feedforward_layernorm_2.weight": "model_3.safetensors", + "model.layers.19.router.proj.weight": "model_3.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model_3.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model_3.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model_3.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model_3.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model_3.safetensors", + "model.layers.19.mlp.up_proj.weight": "model_3.safetensors", + "model.layers.19.mlp.down_proj.weight": "model_3.safetensors", + "model.layers.19.experts.gate_up_proj": "model_3.safetensors", + "model.layers.19.experts.down_proj": "model_3.safetensors", + "model.layers.19.layer_scalar": "model_3.safetensors", + "model.layers.20.input_layernorm.weight": "model_3.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model_3.safetensors", + "model.layers.20.post_feedforward_layernorm.weight": "model_3.safetensors", + "model.layers.20.self_attn.q_norm.weight": "model_3.safetensors", + "model.layers.20.self_attn.k_norm.weight": "model_3.safetensors", + "model.layers.20.pre_feedforward_layernorm.weight": "model_3.safetensors", + "model.layers.20.post_feedforward_layernorm_1.weight": "model_3.safetensors", + "model.layers.20.router.scale": "model_3.safetensors", + "model.layers.20.router.per_expert_scale": "model_3.safetensors", + "model.layers.20.pre_feedforward_layernorm_2.weight": "model_3.safetensors", + "model.layers.20.post_feedforward_layernorm_2.weight": "model_3.safetensors", + "model.layers.20.router.proj.weight": "model_3.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model_3.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model_3.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model_3.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model_3.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model_3.safetensors", + "model.layers.20.mlp.up_proj.weight": "model_3.safetensors", + "model.layers.20.mlp.down_proj.weight": "model_3.safetensors", + "model.layers.20.experts.gate_up_proj": "model_3.safetensors", + "model.layers.20.experts.down_proj": "model_3.safetensors", + "model.layers.20.layer_scalar": "model_4.safetensors", + "model.layers.21.input_layernorm.weight": "model_4.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model_4.safetensors", + "model.layers.21.post_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.21.self_attn.q_norm.weight": "model_4.safetensors", + "model.layers.21.self_attn.k_norm.weight": "model_4.safetensors", + "model.layers.21.pre_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.21.post_feedforward_layernorm_1.weight": "model_4.safetensors", + "model.layers.21.router.scale": "model_4.safetensors", + "model.layers.21.router.per_expert_scale": "model_4.safetensors", + "model.layers.21.pre_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.21.post_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.21.router.proj.weight": "model_4.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model_4.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model_4.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model_4.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model_4.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model_4.safetensors", + "model.layers.21.mlp.up_proj.weight": "model_4.safetensors", + "model.layers.21.mlp.down_proj.weight": "model_4.safetensors", + "model.layers.21.experts.gate_up_proj": "model_4.safetensors", + "model.layers.21.experts.down_proj": "model_4.safetensors", + "model.layers.21.layer_scalar": "model_4.safetensors", + "model.layers.22.input_layernorm.weight": "model_4.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model_4.safetensors", + "model.layers.22.post_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.22.self_attn.q_norm.weight": "model_4.safetensors", + "model.layers.22.self_attn.k_norm.weight": "model_4.safetensors", + "model.layers.22.pre_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.22.post_feedforward_layernorm_1.weight": "model_4.safetensors", + "model.layers.22.router.scale": "model_4.safetensors", + "model.layers.22.router.per_expert_scale": "model_4.safetensors", + "model.layers.22.pre_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.22.post_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.22.router.proj.weight": "model_4.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model_4.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model_4.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model_4.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model_4.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model_4.safetensors", + "model.layers.22.mlp.up_proj.weight": "model_4.safetensors", + "model.layers.22.mlp.down_proj.weight": "model_4.safetensors", + "model.layers.22.experts.gate_up_proj": "model_4.safetensors", + "model.layers.22.experts.down_proj": "model_4.safetensors", + "model.layers.22.layer_scalar": "model_4.safetensors", + "model.layers.23.input_layernorm.weight": "model_4.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model_4.safetensors", + "model.layers.23.post_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.23.self_attn.q_norm.weight": "model_4.safetensors", + "model.layers.23.self_attn.k_norm.weight": "model_4.safetensors", + "model.layers.23.pre_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.23.post_feedforward_layernorm_1.weight": "model_4.safetensors", + "model.layers.23.router.scale": "model_4.safetensors", + "model.layers.23.router.per_expert_scale": "model_4.safetensors", + "model.layers.23.pre_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.23.post_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.23.router.proj.weight": "model_4.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model_4.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model_4.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model_4.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model_4.safetensors", + "model.layers.23.mlp.up_proj.weight": "model_4.safetensors", + "model.layers.23.mlp.down_proj.weight": "model_4.safetensors", + "model.layers.23.experts.gate_up_proj": "model_4.safetensors", + "model.layers.23.experts.down_proj": "model_4.safetensors", + "model.layers.23.layer_scalar": "model_4.safetensors", + "model.layers.24.input_layernorm.weight": "model_4.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model_4.safetensors", + "model.layers.24.post_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.24.self_attn.q_norm.weight": "model_4.safetensors", + "model.layers.24.self_attn.k_norm.weight": "model_4.safetensors", + "model.layers.24.pre_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.24.post_feedforward_layernorm_1.weight": "model_4.safetensors", + "model.layers.24.router.scale": "model_4.safetensors", + "model.layers.24.router.per_expert_scale": "model_4.safetensors", + "model.layers.24.pre_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.24.post_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.24.router.proj.weight": "model_4.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model_4.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model_4.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model_4.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model_4.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model_4.safetensors", + "model.layers.24.mlp.up_proj.weight": "model_4.safetensors", + "model.layers.24.mlp.down_proj.weight": "model_4.safetensors", + "model.layers.24.experts.gate_up_proj": "model_4.safetensors", + "model.layers.24.experts.down_proj": "model_4.safetensors", + "model.layers.24.layer_scalar": "model_4.safetensors", + "model.layers.25.input_layernorm.weight": "model_4.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model_4.safetensors", + "model.layers.25.post_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.25.self_attn.q_norm.weight": "model_4.safetensors", + "model.layers.25.self_attn.k_norm.weight": "model_4.safetensors", + "model.layers.25.pre_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.25.post_feedforward_layernorm_1.weight": "model_4.safetensors", + "model.layers.25.router.scale": "model_4.safetensors", + "model.layers.25.router.per_expert_scale": "model_4.safetensors", + "model.layers.25.pre_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.25.post_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.25.router.proj.weight": "model_4.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model_4.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model_4.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model_4.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model_4.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model_4.safetensors", + "model.layers.25.mlp.up_proj.weight": "model_4.safetensors", + "model.layers.25.mlp.down_proj.weight": "model_4.safetensors", + "model.layers.25.experts.gate_up_proj": "model_4.safetensors", + "model.layers.25.experts.down_proj": "model_4.safetensors", + "model.layers.25.layer_scalar": "model_4.safetensors", + "model.layers.26.input_layernorm.weight": "model_4.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model_4.safetensors", + "model.layers.26.post_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.26.self_attn.q_norm.weight": "model_4.safetensors", + "model.layers.26.self_attn.k_norm.weight": "model_4.safetensors", + "model.layers.26.pre_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.26.post_feedforward_layernorm_1.weight": "model_4.safetensors", + "model.layers.26.router.scale": "model_4.safetensors", + "model.layers.26.router.per_expert_scale": "model_4.safetensors", + "model.layers.26.pre_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.26.post_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.26.router.proj.weight": "model_4.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model_4.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model_4.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model_4.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model_4.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model_4.safetensors", + "model.layers.26.mlp.up_proj.weight": "model_4.safetensors", + "model.layers.26.mlp.down_proj.weight": "model_4.safetensors", + "model.layers.26.experts.gate_up_proj": "model_4.safetensors", + "model.layers.26.experts.down_proj": "model_5.safetensors", + "model.layers.26.layer_scalar": "model_5.safetensors", + "model.layers.27.input_layernorm.weight": "model_5.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model_5.safetensors", + "model.layers.27.post_feedforward_layernorm.weight": "model_5.safetensors", + "model.layers.27.self_attn.q_norm.weight": "model_5.safetensors", + "model.layers.27.self_attn.k_norm.weight": "model_5.safetensors", + "model.layers.27.pre_feedforward_layernorm.weight": "model_5.safetensors", + "model.layers.27.post_feedforward_layernorm_1.weight": "model_5.safetensors", + "model.layers.27.router.scale": "model_5.safetensors", + "model.layers.27.router.per_expert_scale": "model_5.safetensors", + "model.layers.27.pre_feedforward_layernorm_2.weight": "model_5.safetensors", + "model.layers.27.post_feedforward_layernorm_2.weight": "model_5.safetensors", + "model.layers.27.router.proj.weight": "model_5.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model_5.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model_5.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model_5.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model_5.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model_5.safetensors", + "model.layers.27.mlp.up_proj.weight": "model_5.safetensors", + "model.layers.27.mlp.down_proj.weight": "model_5.safetensors", + "model.layers.27.experts.gate_up_proj": "model_5.safetensors", + "model.layers.27.experts.down_proj": "model_5.safetensors", + "model.layers.27.layer_scalar": "model_5.safetensors", + "model.layers.28.input_layernorm.weight": "model_5.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model_5.safetensors", + "model.layers.28.post_feedforward_layernorm.weight": "model_5.safetensors", + "model.layers.28.self_attn.q_norm.weight": "model_5.safetensors", + "model.layers.28.self_attn.k_norm.weight": "model_5.safetensors", + "model.layers.28.pre_feedforward_layernorm.weight": "model_5.safetensors", + "model.layers.28.post_feedforward_layernorm_1.weight": "model_5.safetensors", + "model.layers.28.router.scale": "model_5.safetensors", + "model.layers.28.router.per_expert_scale": "model_5.safetensors", + "model.layers.28.pre_feedforward_layernorm_2.weight": "model_5.safetensors", + "model.layers.28.post_feedforward_layernorm_2.weight": "model_5.safetensors", + "model.layers.28.router.proj.weight": "model_5.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model_5.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model_5.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model_5.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model_5.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model_5.safetensors", + "model.layers.28.mlp.up_proj.weight": "model_5.safetensors", + "model.layers.28.mlp.down_proj.weight": "model_5.safetensors", + "model.layers.28.experts.gate_up_proj": "model_5.safetensors", + "model.layers.28.experts.down_proj": "model_5.safetensors", + "model.layers.28.layer_scalar": "model_5.safetensors", + "model.layers.29.input_layernorm.weight": "model_5.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model_5.safetensors", + "model.layers.29.post_feedforward_layernorm.weight": "model_5.safetensors", + "model.layers.29.self_attn.q_norm.weight": "model_5.safetensors", + "model.layers.29.self_attn.k_norm.weight": "model_5.safetensors", + "model.layers.29.pre_feedforward_layernorm.weight": "model_5.safetensors", + "model.layers.29.post_feedforward_layernorm_1.weight": "model_5.safetensors", + "model.layers.29.router.scale": "model_5.safetensors", + "model.layers.29.router.per_expert_scale": "model_5.safetensors", + "model.layers.29.pre_feedforward_layernorm_2.weight": "model_5.safetensors", + "model.layers.29.post_feedforward_layernorm_2.weight": "model_5.safetensors", + "model.layers.29.router.proj.weight": "model_5.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model_5.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model_5.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model_5.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model_5.safetensors", + "model.layers.29.mlp.up_proj.weight": "model_5.safetensors", + "model.layers.29.mlp.down_proj.weight": "model_5.safetensors", + "model.layers.29.experts.gate_up_proj": "model_5.safetensors", + "model.layers.29.experts.down_proj": "model_5.safetensors", + "model.layers.29.layer_scalar": "model_5.safetensors", + "model.norm.weight": "model_5.safetensors" + } +} \ No newline at end of file diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/140/model_0.safetensors b/26_09_08_sft_email_v4_v3/export/gemma4/140/model_0.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5cd73774e7beefef74eca9691437659904701120 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/140/model_0.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7277a2f42a87e19765332e8d65f5f57a4fc8cd9f4fa0dbaa4c48f5b110544a13 +size 18219689440 diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/140/model_1.safetensors b/26_09_08_sft_email_v4_v3/export/gemma4/140/model_1.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3b7d019b45261f6c9817c92eafebabc3f722632f --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/140/model_1.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:204e8a7fe92c63083155d33933c1a6da90188f699f6e0a9836bef21c3b14098c +size 17354616396 diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/140/model_2.safetensors b/26_09_08_sft_email_v4_v3/export/gemma4/140/model_2.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d94e6cd1eaadcac07626bcae46d18888d6265773 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/140/model_2.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6230bf7ac58d23004a10d04b2a8a02da00323c198337ade4ab69e97bb9355f02 +size 18580955576 diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/140/model_3.safetensors b/26_09_08_sft_email_v4_v3/export/gemma4/140/model_3.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..206e0b8ade69a90fbc59e96de04631ff745624eb --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/140/model_3.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d3a2cad124dd0352a5285efc72021131105b10f3fe8d1c2eaa8ab988da7133e +size 17354616508 diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/140/model_4.safetensors b/26_09_08_sft_email_v4_v3/export/gemma4/140/model_4.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..259b64b21f6f10ade91c597a3a515166526cc41b --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/140/model_4.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49322496e747f7c8f2535553d59da3efef60dc07cd9181b908c6d3515b1c00eb +size 18580955560 diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/140/model_5.safetensors b/26_09_08_sft_email_v4_v3/export/gemma4/140/model_5.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7983055675e5074341cf54c1c18d391f0adb70d2 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/140/model_5.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:838deccb7a04bf4e7b1dcbf8eaf043dc0b0dbbb2b775073e19e1f352ee86aaba +size 10841863008 diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/140/ok b/26_09_08_sft_email_v4_v3/export/gemma4/140/ok new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/140/tokenizer.json b/26_09_08_sft_email_v4_v3/export/gemma4/140/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..1ff9f3e3439a939b971f9919e821bf87e835a503 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/140/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc8d3a0ce36466ccc1278bf987df5f71db1719b9ca6b4118264f45cb627bfe0f +size 32169626 diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/140/tokenizer_config.json b/26_09_08_sft_email_v4_v3/export/gemma4/140/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..375b25dc8be85705251e41be1c25310d24932051 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/140/tokenizer_config.json @@ -0,0 +1,74 @@ +{ + "audio_token": "<|audio|>", + "backend": "tokenizers", + "boa_token": "<|audio>", + "boi_token": "<|image>", + "bos_token": "", + "eoa_token": "", + "eoc_token": "", + "eoi_token": "", + "eos_token": "", + "eot_token": "", + "escape_token": "<|\"|>", + "etc_token": "", + "etd_token": "", + "etr_token": "", + "extra_special_tokens": [ + "<|video|>" + ], + "image_token": "<|image|>", + "mask_token": "", + "model_max_length": 1000000000000000019884624838656, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma4Processor", + "response_schema": { + "type": "object", + "properties": { + "role": { + "const": "assistant" + }, + "thinking": { + "type": "string" + }, + "content": { + "type": "string" + }, + "tool_calls": { + "x-regex-iterator": "<\\|tool_call>(.*?)", + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "const": "function" + }, + "function": { + "type": "object", + "x-regex": "call\\:(?P\\w+)(?P\\{.*\\})", + "properties": { + "name": { + "type": "string" + }, + "arguments": { + "type": "object", + "x-parser": "gemma4-tool-call", + "additionalProperties": {} + } + } + } + } + } + } + }, + "x-regex": "(\\<\\|channel\\>thought\\n(?P.*?)\\)?(?P\\<\\|tool_call\\>.*\\)?(?P(?:(?!\\)(?!\\<\\|tool_response\\>).)+)?(?:\\|\\<\\|tool_response\\>)?" + }, + "soc_token": "<|channel>", + "sot_token": "<|turn>", + "stc_token": "<|tool_call>", + "std_token": "<|tool>", + "str_token": "<|tool_response>", + "think_token": "<|think|>", + "tokenizer_class": "GemmaTokenizer", + "unk_token": "" +} diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/210/chat_template.jinja b/26_09_08_sft_email_v4_v3/export/gemma4/210/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..20e0028946060065b95e9376890a07f52b3c7b6f --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/210/chat_template.jinja @@ -0,0 +1,390 @@ +{# + Template: Google Gemma 4 Canonical Chat Template + Author: Google Gemma Engineering Team + Published: 2026-07-09 + Context: Fixed tool-calling loops, turn closures, and thinking content-ordering. +#} +{%- macro format_parameters(properties, required, filter_keys=false) -%} + {%- set standard_keys = ['properties', 'required', 'nullable'] -%} + {%- set ns = namespace(found_first=false) -%} + {%- for key, value in properties | dictsort -%} + {%- set add_comma = false -%} + {%- if not filter_keys or key not in standard_keys -%} + {%- if ns.found_first %},{% endif -%} + {%- set ns.found_first = true -%} + {{ key }}:{ + {%- if value['description'] -%} + description:<|"|>{{ value['description'] }}<|"|> + {%- set add_comma = true -%} + {%- endif -%} + {%- if value['type'] | upper == 'STRING' -%} + {%- if value['enum'] -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + enum:{{ format_argument(value['enum']) }} + {%- endif -%} + {%- elif value['type'] | upper == 'ARRAY' -%} + {%- if value['items'] is mapping and value['items'] -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + items:{ + {%- set ns_items = namespace(found_first=false) -%} + {%- for item_key, item_value in value['items'] | dictsort -%} + {%- if item_value is not none -%} + {%- if ns_items.found_first %},{% endif -%} + {%- set ns_items.found_first = true -%} + {%- if item_key == 'properties' -%} + properties:{ + {%- if item_value is mapping -%} + {{- format_parameters(item_value, value['items']['required'] | default([])) -}} + {%- endif -%} + } + {%- elif item_key == 'required' -%} + required:[ + {%- for req_item in item_value -%} + <|"|>{{- req_item -}}<|"|> + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + ] + {%- elif item_key == 'type' -%} + {%- if item_value is string -%} + type:{{ format_argument(item_value | upper) }} + {%- else -%} + type:{{ format_argument(item_value | map('upper') | list) }} + {%- endif -%} + {%- else -%} + {{ item_key }}:{{ format_argument(item_value) }} + {%- endif -%} + {%- endif -%} + {%- endfor -%} + } + {%- endif -%} + {%- endif -%} + {%- if value['nullable'] %} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + nullable:true + {%- endif -%} + {%- if value['type'] | upper == 'OBJECT' -%} + {%- if value['properties'] is defined and value['properties'] is mapping -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + properties:{ + {{- format_parameters(value['properties'], value['required'] | default([])) -}} + } + {%- elif value is mapping -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + properties:{ + {{- format_parameters(value, value['required'] | default([]), filter_keys=true) -}} + } + {%- endif -%} + {%- if value['required'] -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + required:[ + {%- for item in value['required'] | default([]) -%} + <|"|>{{- item -}}<|"|> + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + ] + {%- endif -%} + {%- endif -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + type:<|"|>{{ value['type'] | upper }}<|"|>} + {%- endif -%} + {%- endfor -%} +{%- endmacro -%} +{%- macro format_function_declaration(tool_data) -%} + declaration:{{- tool_data['function']['name'] -}}{description:<|"|>{{- tool_data['function']['description'] -}}<|"|> + {%- set params = tool_data['function']['parameters'] -%} + {%- if params -%} + ,parameters:{ + {%- if params['properties'] -%} + properties:{ {{- format_parameters(params['properties'], params['required']) -}} }, + {%- endif -%} + {%- if params['required'] -%} + required:[ + {%- for item in params['required'] -%} + <|"|>{{- item -}}<|"|> + {{- ',' if not loop.last -}} + {%- endfor -%} + ], + {%- endif -%} + {%- if params['type'] -%} + type:<|"|>{{- params['type'] | upper -}}<|"|>} + {%- endif -%} + {%- endif -%} + {%- if 'response' in tool_data['function'] -%} + {%- set response_declaration = tool_data['function']['response'] -%} + ,response:{ + {%- if response_declaration['description'] -%} + description:<|"|>{{- response_declaration['description'] -}}<|"|>, + {%- endif -%} + {%- if response_declaration['type'] | upper == 'OBJECT' -%} + type:<|"|>{{- response_declaration['type'] | upper -}}<|"|>} + {%- endif -%} + {%- endif -%} + } +{%- endmacro -%} +{%- macro format_argument(argument, escape_keys=True) -%} + {%- if argument is none -%} + {{- 'null' -}} + {%- elif argument is string -%} + {{- '<|"|>' + argument + '<|"|>' -}} + {%- elif argument is boolean -%} + {{- 'true' if argument else 'false' -}} + {%- elif argument is mapping -%} + {{- '{' -}} + {%- set ns = namespace(found_first=false) -%} + {%- for key, value in argument | dictsort -%} + {%- if ns.found_first %},{% endif -%} + {%- set ns.found_first = true -%} + {%- if escape_keys -%} + {{- '<|"|>' + key + '<|"|>' -}} + {%- else -%} + {{- key -}} + {%- endif -%} + :{{- format_argument(value, escape_keys=escape_keys) -}} + {%- endfor -%} + {{- '}' -}} + {%- elif argument is sequence -%} + {{- '[' -}} + {%- for item in argument -%} + {{- format_argument(item, escape_keys=escape_keys) -}} + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + {{- ']' -}} + {%- else -%} + {{- argument -}} + {%- endif -%} +{%- endmacro -%} +{%- macro strip_thinking(text) -%} + {%- set ns = namespace(result='') -%} + {%- for part in text.split('') -%} + {%- if '<|channel>' in part -%} + {%- set ns.result = ns.result + part.split('<|channel>')[0] -%} + {%- else -%} + {%- set ns.result = ns.result + part -%} + {%- endif -%} + {%- endfor -%} + {{- ns.result | trim -}} +{%- endmacro -%} + +{%- macro format_tool_response_block(tool_name, response) -%} + {{- '<|tool_response>' -}} + {%- if response is mapping -%} + {{- 'response:' + tool_name + '{' -}} + {%- for key, value in response | dictsort -%} + {{- key -}}:{{- format_argument(value, escape_keys=False) -}} + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + {{- '}' -}} + {%- else -%} + {{- 'response:' + tool_name + '{value:' + format_argument(response, escape_keys=False) + '}' -}} + {%- endif -%} + {{- '' -}} +{%- endmacro -%} + +{#- ===== SETUP ===== -#} +{%- set ns = namespace(prev_message_type=None, prev_non_tool_role=None) -%} +{%- set loop_messages = messages -%} +{%- set enable_thinking = enable_thinking | default(false) -%} +{%- set preserve_thinking = preserve_thinking | default(false) -%} +{{- bos_token -}} +{#- Handle System/Tool Definitions Block -#} +{%- if enable_thinking or tools or (messages and messages[0]['role'] in ['system', 'developer']) -%} + {{- '<|turn>system\n' -}} + {#- Inject Thinking token at the very top of the FIRST system turn -#} + {%- if enable_thinking -%} + {{- '<|think|>\n' -}} + {%- set ns.prev_message_type = 'think' -%} + {%- endif -%} + {%- if messages and messages[0]['role'] in ['system', 'developer'] -%} + {%- if messages[0]['content'] is string -%} + {{- messages[0]['content'] | trim -}} + {%- elif messages[0]['content'] is sequence -%} + {%- for item in messages[0]['content'] -%} + {{- item['text'] | trim + ' '-}} + {%- endfor -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} + {%- endif -%} + {%- if tools -%} + {%- for tool in tools %} + {{- '<|tool>' -}} + {{- format_function_declaration(tool) | trim -}} + {{- '' -}} + {%- endfor %} + {%- set ns.prev_message_type = 'tool' -%} + {%- endif -%} + {{- '\n' -}} +{%- endif %} + +{#- Pre-scan: find last user message index for reasoning guard -#} +{%- set ns_turn = namespace(last_user_idx=-1) -%} +{%- for i in range(loop_messages | length) -%} + {%- if loop_messages[i]['role'] == 'user' -%} + {%- set ns_turn.last_user_idx = i -%} + {%- endif -%} +{%- endfor -%} + +{#- Loop through messages -#} +{%- for message in loop_messages -%} + {%- if message['role'] != 'tool' -%} + {%- set ns.prev_message_type = None -%} + {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%} + {#- Detect continuation using tracked state — O(1) instead of O(n) backward scan -#} + {%- set continue_same_model_turn = (role == 'model' and ns.prev_non_tool_role == 'assistant') -%} + {%- if not continue_same_model_turn -%} + {{- '<|turn>' + role + '\n' }} + + {%- endif -%} + + {#- Render reasoning/reasoning_content as thinking channel -#} + {%- set thinking_text = message.get('reasoning') or message.get('reasoning_content') -%} + {%- set thinking_gate = (loop.index0 > ns_turn.last_user_idx) or (preserve_thinking and message.get('tool_calls')) -%} + {%- if thinking_text and thinking_gate -%} + {{- '<|channel>thought\n' + thinking_text + '\n' -}} + {%- endif -%} + + {%- if message.get('tool_calls') -%} + {%- for tool_call in message.get('tool_calls') -%} + {%- set function = tool_call['function'] -%} + {{- '<|tool_call>call:' + function['name'] + '{' -}} + {%- if function['arguments'] is mapping -%} + {%- set ns_args = namespace(found_first=false) -%} + {%- for key, value in function['arguments'] | dictsort -%} + {%- if ns_args.found_first %},{% endif -%} + {%- set ns_args.found_first = true -%} + {{- key -}}:{{- format_argument(value, escape_keys=False) -}} + {%- endfor -%} + {%- elif function['arguments'] is none -%} + {%- else -%} + {{- raise_exception( + "chat_template: tool_calls[].function.arguments must be a " + "JSON object (mapping), not a string. Deserialize arguments " + "before passing to the template." + ) -}} + {%- endif -%} + {{- '}' -}} + {%- endfor -%} + {%- set ns.prev_message_type = 'tool_call' -%} + {%- endif -%} + + {%- set ns_tr_out = namespace(flag=false) -%} + {%- if message.get('tool_responses') -%} + {#- Legacy: tool_responses embedded on the assistant message (Google/Gemma native) -#} + {%- for tool_response in message.get('tool_responses') -%} + {{- format_tool_response_block(tool_response['name'] | default('unknown', true), tool_response['response']) -}} + {%- set ns_tr_out.flag = true -%} + {%- set ns.prev_message_type = 'tool_response' -%} + {%- endfor -%} + {%- elif message.get('tool_calls') -%} + {#- OpenAI Chat Completions: forward-scan consecutive role:tool messages -#} + {%- set ns_tool_scan = namespace(stopped=false) -%} + {%- for k in range(loop.index0 + 1, loop_messages | length) -%} + {%- if ns_tool_scan.stopped -%} + {%- elif loop_messages[k]['role'] != 'tool' -%} + {%- set ns_tool_scan.stopped = true -%} + {%- else -%} + {%- set follow = loop_messages[k] -%} + {#- Resolve tool_call_id to function name -#} + {%- set ns_tname = namespace(name=follow.get('name') or 'unknown') -%} + {%- for tc in message.get('tool_calls') -%} + {%- if tc.get('id') == follow.get('tool_call_id') -%} + {%- set ns_tname.name = tc['function']['name'] -%} + {%- endif -%} + {%- endfor -%} + {#- Handle content as string or content-parts array -#} + {%- set tool_body = follow.get('content') -%} + {%- if tool_body is string -%} + {{- format_tool_response_block(ns_tname.name, tool_body) -}} + {%- elif tool_body is sequence and tool_body is not string -%} + {%- set ns_txt = namespace(s='') -%} + {%- for part in tool_body -%} + {%- if part.get('type') == 'text' -%} + {%- set ns_txt.s = ns_txt.s + (part.get('text') | default('')) -%} + {%- endif -%} + {%- endfor -%} + {{- format_tool_response_block(ns_tname.name, ns_txt.s) -}} + {%- for part in tool_body -%} + {%- if part.get('type') in ['image', 'image_url'] -%} + {{- '<|image|>' -}} + {%- elif part.get('type') in ['audio', 'input_audio'] -%} + {{- '<|audio|>' -}} + {%- elif part.get('type') == 'video' -%} + {{- '<|video|>' -}} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{- format_tool_response_block(ns_tname.name, tool_body) -}} + {%- endif -%} + {%- set ns_tr_out.flag = true -%} + {%- set ns.prev_message_type = 'tool_response' -%} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + + {%- set captured_content -%} + {%- if message.get('content') is string -%} + {%- if role == 'model' -%} + {{- strip_thinking(message['content']) -}} + {%- else -%} + {{- message['content'] | trim -}} + {%- endif -%} + {%- elif message.get('content') is sequence -%} + {%- for item in message['content'] -%} + {%- if item.get('type') == 'text' -%} + {%- if role == 'model' -%} + {{- strip_thinking(item['text']) -}} + {%- else -%} + {{- item['text'] | trim -}} + {%- endif -%} + {%- elif item.get('type') in ['image', 'image_url'] -%} + {{- '<|image|>' -}} + {%- elif item.get('type') in ['audio', 'input_audio'] -%} + {{- '<|audio|>' -}} + {%- elif item.get('type') == 'video' -%} + {{- '<|video|>' -}} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + {%- endset -%} + + {{- captured_content -}} + {%- set has_content = captured_content | trim | length > 0 -%} + + {#- Forward-scan: find next non-tool message role for continuation detection -#} + {%- set next_nt = namespace(role=None, found=false) -%} + {%- for j in range(loop.index0 + 1, loop_messages | length) -%} + {%- if not next_nt.found -%} + {%- if loop_messages[j]['role'] != 'tool' -%} + {%- set next_nt.role = loop_messages[j]['role'] -%} + {%- set next_nt.found = true -%} + {%- endif -%} + {%- endif -%} + {%- endfor -%} + + {%- set continues_into_next = ( + role == 'model' + and next_nt.role == 'assistant' + and (not message.get('tool_calls') or ns_tr_out.flag) + ) -%} + + {%- if ns.prev_message_type == 'tool_call' and not ns_tr_out.flag -%} + {{- '<|tool_response>' -}} + {%- elif continues_into_next -%} + {%- elif not (ns_tr_out.flag and not has_content and not next_nt.found) -%} + {{- '\n' -}} + {%- endif -%} + + {#- Track previous non-tool role for next iteration (avoids O(n) backward scan) -#} + {%- set ns.prev_non_tool_role = message['role'] -%} + {%- endif -%} +{%- endfor -%} + +{%- if add_generation_prompt -%} + {%- if ns.prev_message_type != 'tool_response' and ns.prev_message_type != 'tool_call' -%} + {{- '<|turn>model\n' -}} + {%- if not enable_thinking -%} + {{- '<|channel>thought\n' -}} + {%- endif -%} + {%- elif ns.prev_message_type == 'tool_response' and enable_thinking -%} + {{- '<|channel>thought\n' -}} + {%- endif -%} +{%- endif -%} diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/210/config.json b/26_09_08_sft_email_v4_v3/export/gemma4/210/config.json new file mode 100644 index 0000000000000000000000000000000000000000..518607d204ad0bf72d56220356c72e1516d05041 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/210/config.json @@ -0,0 +1,83 @@ +{ + "attention_bias": false, + "attention_dropout": 0.0, + "attention_k_eq_v": true, + "bos_token_id": 2, + "dtype": "bfloat16", + "enable_moe_block": true, + "eos_token_id": 1, + "final_logit_softcapping": 30.0, + "global_head_dim": 512, + "head_dim": 256, + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 2816, + "hidden_size_per_layer_input": 0, + "initializer_range": 0.02, + "intermediate_size": 2112, + "layer_types": [ + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 262144, + "model_type": "gemma4_text", + "moe_intermediate_size": 704, + "num_attention_heads": 16, + "num_experts": 128, + "num_global_key_value_heads": 2, + "num_hidden_layers": 30, + "num_key_value_heads": 8, + "num_kv_shared_layers": 0, + "pad_token_id": 0, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "full_attention": { + "partial_rotary_factor": 0.25, + "rope_theta": 1000000.0, + "rope_type": "proportional" + }, + "sliding_attention": { + "rope_theta": 10000.0, + "rope_type": "default" + } + }, + "sliding_window": 1024, + "tie_word_embeddings": true, + "top_k_experts": 8, + "use_bidirectional_attention": "vision", + "use_cache": true, + "use_double_wide_mlp": false, + "vocab_size": 262144, + "vocab_size_per_layer_input": 262144, + "architectures": [ + "Gemma4ForCausalLM" + ], + "transformers_version": "5.5.0.dev0" +} \ No newline at end of file diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/210/generation_config.json b/26_09_08_sft_email_v4_v3/export/gemma4/210/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e605bb4523b1462ea9d9a3810b9e3ecf7ab7b1f6 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/210/generation_config.json @@ -0,0 +1,14 @@ +{ + "bos_token_id": 2, + "do_sample": true, + "eos_token_id": [ + 1, + 106, + 50 + ], + "pad_token_id": 0, + "temperature": 1.0, + "top_k": 64, + "top_p": 0.95, + "transformers_version": "5.5.0.dev0" +} diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/210/model.safetensors.index.json b/26_09_08_sft_email_v4_v3/export/gemma4/210/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..97ea655695d295f0601119aae065eb0c635096dd --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/210/model.safetensors.index.json @@ -0,0 +1,1053 @@ +{ + "metadata": { + "fast_llm_metadata": { + "fast_llm_version": "0.4.0", + "model": "gpt", + "format": "gemma4", + "config": { + "type": "gpt", + "base_model": { + "decoder": { + "type": "pattern", + "blocks": { + "full_attention": { + "type": "decoder", + "mixer": { + "type": "attention", + "rotary": { + "type": "proportional", + "theta": 1000000.0, + "partial_rotary_factor": 0.25 + }, + "heads": 16, + "head_groups": 2, + "head_size": 512, + "add_linear_biases": false, + "dropout": 0.0, + "softmax_scale_power": 0.0, + "implementation": "sdpa", + "query_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "key_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "value_norm": { + "type": "fixed_rms_norm", + "epsilon": 1e-06 + }, + "shared_key_value": true + }, + "mlp": { + "type": "hybrid_moe", + "dense": { + "type": "mlp", + "pre_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "post_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "intermediate_size": 2112, + "add_linear_biases": false, + "gated": true, + "activation": "gelu", + "recompute_level": "full" + }, + "routed": { + "type": "moe", + "pre_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "post_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "intermediate_size": 704, + "add_linear_biases": false, + "gated": true, + "activation": "gelu", + "recompute_level": "full", + "router_normalization": { + "type": "fixed_rms_norm", + "epsilon": 1e-06 + }, + "router_scale": { + "enabled": true + }, + "router_input_scale": 0.018844459036110227, + "router_per_expert_scale": { + "enabled": true + }, + "experts": 128, + "experts_per_token": 8, + "auxiliary_loss_coefficient": 0.0 + } + }, + "normalization": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "pre_mlp_normalization": { + "type": "none" + }, + "post_mixer_normalization": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "post_mlp_normalization": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "output_scale": { + "lr_scale": 0.0, + "enabled": true + } + }, + "sliding_attention": { + "type": "decoder", + "mixer": { + "type": "attention", + "rotary": { + "type": "default", + "theta": 10000.0 + }, + "heads": 16, + "head_groups": 8, + "head_size": 256, + "add_linear_biases": false, + "dropout": 0.0, + "window_size": 1024, + "softmax_scale_power": 0.0, + "implementation": "flash", + "query_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "key_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "value_norm": { + "type": "fixed_rms_norm", + "epsilon": 1e-06 + } + }, + "mlp": { + "type": "hybrid_moe", + "dense": { + "type": "mlp", + "pre_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "post_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "intermediate_size": 2112, + "add_linear_biases": false, + "gated": true, + "activation": "gelu", + "recompute_level": "full" + }, + "routed": { + "type": "moe", + "pre_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "post_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "intermediate_size": 704, + "add_linear_biases": false, + "gated": true, + "activation": "gelu", + "recompute_level": "full", + "router_normalization": { + "type": "fixed_rms_norm", + "epsilon": 1e-06 + }, + "router_scale": { + "enabled": true + }, + "router_input_scale": 0.018844459036110227, + "router_per_expert_scale": { + "enabled": true + }, + "experts": 128, + "experts_per_token": 8, + "auxiliary_loss_coefficient": 0.0 + } + }, + "normalization": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "pre_mlp_normalization": { + "type": "none" + }, + "post_mixer_normalization": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "post_mlp_normalization": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "output_scale": { + "lr_scale": 0.0, + "enabled": true + } + } + }, + "pattern": [ + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention" + ], + "num_blocks": 30 + }, + "embeddings": { + "vocab_size": 262144, + "vocab_parallel": false, + "embedding_scale": 53.0659966456864 + }, + "head": { + "normalization": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "cross_entropy_splits": 8, + "final_logit_softcap": 30.0 + }, + "tied_embedding_weight": true, + "hidden_size": 2816, + "peft": { + "type": "none" + } + }, + "multi_stage": { + "zero_stage": 3 + }, + "distributed": { + "tensor_parallel": 2, + "sequence_tensor_parallel": true, + "sequence_data_parallel": 4, + "world_size": 32, + "rank": 0, + "local_world_size": 8, + "timeout": 3600.0, + "compute_dtype": "bfloat16" + } + }, + "shards": [ + "weights" + ], + "metadata": { + "optimizer": { + "current_step": 210, + "grad_scaler": { + "type": "NoopGradScaler" + } + }, + "completed_steps": 210, + "metrics": { + "training": { + "batch_size": 2097152, + "load_balancing_loss": 815.46337890625, + "lm_head_loss": 0.11222395300865173, + "total_steps": 210, + "completed_steps": 210, + "consumed_tokens": 440401920, + "percent_done": 100.0, + "step_time_ms": 28241.050300188363, + "step_time_average_ms": 30303.581654688434, + "remaining_time": 0.0, + "completion_time": 1788873090.3226085, + "skipped_iters": 0, + "nan_iters": 0, + "model_tflops": 59447.884978233706, + "hardware_tflops": 73753.52245463323, + "tokens_per_sec_per_gpu": 2320.5935793245935, + "run": 0, + "grad_norm": 2.482532262802124, + "learning_rate": 0.0, + "loss_scale": 1.0, + "reserved": 55562.0, + "allocated": 21478.49755859375, + "max_allocated": 54800.1123046875, + "max_reserved": 55562.0, + "global_max_reserved": 74782.0 + } + } + } + }, + "model_config": { + "vocab_size": 262144, + "num_attention_heads": 16, + "num_key_value_heads": 8, + "head_dim": 256, + "global_head_dim": 512, + "num_global_key_value_heads": 2, + "attention_bias": false, + "attention_dropout": 0.0, + "sliding_window": 1024, + "rms_norm_eps": 1e-06, + "attention_k_eq_v": true, + "rope_parameters": { + "sliding_attention": { + "rope_type": "default", + "rope_theta": 10000.0 + }, + "full_attention": { + "rope_type": "proportional", + "rope_theta": 1000000.0, + "partial_rotary_factor": 0.25 + } + }, + "intermediate_size": 2112, + "hidden_activation": "gelu_pytorch_tanh", + "num_experts": 128, + "top_k_experts": 8, + "moe_intermediate_size": 704, + "enable_moe_block": true, + "num_hidden_layers": 30, + "layer_types": [ + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention" + ], + "final_logit_softcapping": 30.0, + "tie_word_embeddings": true, + "hidden_size": 2816, + "hidden_size_per_layer_input": 0, + "use_bidirectional_attention": null, + "model_type": "gemma4_text", + "architectures": [ + "Gemma4ForCausalLM" + ] + }, + "format": "pt" + }, + "weight_map": { + "model.embed_tokens.weight": "model_0.safetensors", + "model.layers.0.input_layernorm.weight": "model_0.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model_0.safetensors", + "model.layers.0.post_feedforward_layernorm.weight": "model_0.safetensors", + "model.layers.0.self_attn.q_norm.weight": "model_0.safetensors", + "model.layers.0.self_attn.k_norm.weight": "model_0.safetensors", + "model.layers.0.pre_feedforward_layernorm.weight": "model_0.safetensors", + "model.layers.0.post_feedforward_layernorm_1.weight": "model_0.safetensors", + "model.layers.0.router.scale": "model_0.safetensors", + "model.layers.0.router.per_expert_scale": "model_0.safetensors", + "model.layers.0.pre_feedforward_layernorm_2.weight": "model_0.safetensors", + "model.layers.0.post_feedforward_layernorm_2.weight": "model_0.safetensors", + "model.layers.0.router.proj.weight": "model_0.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model_0.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model_0.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model_0.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model_0.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model_0.safetensors", + "model.layers.0.mlp.up_proj.weight": "model_0.safetensors", + "model.layers.0.mlp.down_proj.weight": "model_0.safetensors", + "model.layers.0.experts.gate_up_proj": "model_0.safetensors", + "model.layers.0.experts.down_proj": "model_0.safetensors", + "model.layers.0.layer_scalar": "model_0.safetensors", + "model.layers.1.input_layernorm.weight": "model_0.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model_0.safetensors", + "model.layers.1.post_feedforward_layernorm.weight": "model_0.safetensors", + "model.layers.1.self_attn.q_norm.weight": "model_0.safetensors", + "model.layers.1.self_attn.k_norm.weight": "model_0.safetensors", + "model.layers.1.pre_feedforward_layernorm.weight": "model_0.safetensors", + "model.layers.1.post_feedforward_layernorm_1.weight": "model_0.safetensors", + "model.layers.1.router.scale": "model_0.safetensors", + "model.layers.1.router.per_expert_scale": "model_0.safetensors", + "model.layers.1.pre_feedforward_layernorm_2.weight": "model_0.safetensors", + "model.layers.1.post_feedforward_layernorm_2.weight": "model_0.safetensors", + "model.layers.1.router.proj.weight": "model_0.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model_0.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model_0.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model_0.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model_0.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model_0.safetensors", + "model.layers.1.mlp.up_proj.weight": "model_0.safetensors", + "model.layers.1.mlp.down_proj.weight": "model_0.safetensors", + "model.layers.1.experts.gate_up_proj": "model_0.safetensors", + "model.layers.1.experts.down_proj": "model_0.safetensors", + "model.layers.1.layer_scalar": "model_0.safetensors", + "model.layers.2.input_layernorm.weight": "model_0.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model_0.safetensors", + "model.layers.2.post_feedforward_layernorm.weight": "model_0.safetensors", + "model.layers.2.self_attn.q_norm.weight": "model_0.safetensors", + "model.layers.2.self_attn.k_norm.weight": "model_0.safetensors", + "model.layers.2.pre_feedforward_layernorm.weight": "model_0.safetensors", + "model.layers.2.post_feedforward_layernorm_1.weight": "model_0.safetensors", + "model.layers.2.router.scale": "model_0.safetensors", + "model.layers.2.router.per_expert_scale": "model_0.safetensors", + "model.layers.2.pre_feedforward_layernorm_2.weight": "model_0.safetensors", + "model.layers.2.post_feedforward_layernorm_2.weight": "model_0.safetensors", + "model.layers.2.router.proj.weight": "model_0.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model_0.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model_0.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model_0.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model_0.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model_0.safetensors", + "model.layers.2.mlp.up_proj.weight": "model_0.safetensors", + "model.layers.2.mlp.down_proj.weight": "model_0.safetensors", + "model.layers.2.experts.gate_up_proj": "model_0.safetensors", + "model.layers.2.experts.down_proj": "model_0.safetensors", + "model.layers.2.layer_scalar": "model_0.safetensors", + "model.layers.3.input_layernorm.weight": "model_0.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model_0.safetensors", + "model.layers.3.post_feedforward_layernorm.weight": "model_0.safetensors", + "model.layers.3.self_attn.q_norm.weight": "model_0.safetensors", + "model.layers.3.self_attn.k_norm.weight": "model_0.safetensors", + "model.layers.3.pre_feedforward_layernorm.weight": "model_0.safetensors", + "model.layers.3.post_feedforward_layernorm_1.weight": "model_0.safetensors", + "model.layers.3.router.scale": "model_0.safetensors", + "model.layers.3.router.per_expert_scale": "model_0.safetensors", + "model.layers.3.pre_feedforward_layernorm_2.weight": "model_0.safetensors", + "model.layers.3.post_feedforward_layernorm_2.weight": "model_0.safetensors", + "model.layers.3.router.proj.weight": "model_0.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model_0.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model_0.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model_0.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model_0.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model_0.safetensors", + "model.layers.3.mlp.up_proj.weight": "model_0.safetensors", + "model.layers.3.mlp.down_proj.weight": "model_0.safetensors", + "model.layers.3.experts.gate_up_proj": "model_0.safetensors", + "model.layers.3.experts.down_proj": "model_0.safetensors", + "model.layers.3.layer_scalar": "model_0.safetensors", + "model.layers.4.input_layernorm.weight": "model_0.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model_0.safetensors", + "model.layers.4.post_feedforward_layernorm.weight": "model_0.safetensors", + "model.layers.4.self_attn.q_norm.weight": "model_0.safetensors", + "model.layers.4.self_attn.k_norm.weight": "model_0.safetensors", + "model.layers.4.pre_feedforward_layernorm.weight": "model_0.safetensors", + "model.layers.4.post_feedforward_layernorm_1.weight": "model_0.safetensors", + "model.layers.4.router.scale": "model_0.safetensors", + "model.layers.4.router.per_expert_scale": "model_0.safetensors", + "model.layers.4.pre_feedforward_layernorm_2.weight": "model_0.safetensors", + "model.layers.4.post_feedforward_layernorm_2.weight": "model_0.safetensors", + "model.layers.4.router.proj.weight": "model_0.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model_0.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model_0.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model_0.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model_0.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model_0.safetensors", + "model.layers.4.mlp.up_proj.weight": "model_0.safetensors", + "model.layers.4.mlp.down_proj.weight": "model_0.safetensors", + "model.layers.4.experts.gate_up_proj": "model_0.safetensors", + "model.layers.4.experts.down_proj": "model_1.safetensors", + "model.layers.4.layer_scalar": "model_1.safetensors", + "model.layers.5.input_layernorm.weight": "model_1.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model_1.safetensors", + "model.layers.5.post_feedforward_layernorm.weight": "model_1.safetensors", + "model.layers.5.self_attn.q_norm.weight": "model_1.safetensors", + "model.layers.5.self_attn.k_norm.weight": "model_1.safetensors", + "model.layers.5.pre_feedforward_layernorm.weight": "model_1.safetensors", + "model.layers.5.post_feedforward_layernorm_1.weight": "model_1.safetensors", + "model.layers.5.router.scale": "model_1.safetensors", + "model.layers.5.router.per_expert_scale": "model_1.safetensors", + "model.layers.5.pre_feedforward_layernorm_2.weight": "model_1.safetensors", + "model.layers.5.post_feedforward_layernorm_2.weight": "model_1.safetensors", + "model.layers.5.router.proj.weight": "model_1.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model_1.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model_1.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model_1.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model_1.safetensors", + "model.layers.5.mlp.up_proj.weight": "model_1.safetensors", + "model.layers.5.mlp.down_proj.weight": "model_1.safetensors", + "model.layers.5.experts.gate_up_proj": "model_1.safetensors", + "model.layers.5.experts.down_proj": "model_1.safetensors", + "model.layers.5.layer_scalar": "model_1.safetensors", + "model.layers.6.input_layernorm.weight": "model_1.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model_1.safetensors", + "model.layers.6.post_feedforward_layernorm.weight": "model_1.safetensors", + "model.layers.6.self_attn.q_norm.weight": "model_1.safetensors", + "model.layers.6.self_attn.k_norm.weight": "model_1.safetensors", + "model.layers.6.pre_feedforward_layernorm.weight": "model_1.safetensors", + "model.layers.6.post_feedforward_layernorm_1.weight": "model_1.safetensors", + "model.layers.6.router.scale": "model_1.safetensors", + "model.layers.6.router.per_expert_scale": "model_1.safetensors", + "model.layers.6.pre_feedforward_layernorm_2.weight": "model_1.safetensors", + "model.layers.6.post_feedforward_layernorm_2.weight": "model_1.safetensors", + "model.layers.6.router.proj.weight": "model_1.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model_1.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model_1.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model_1.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model_1.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model_1.safetensors", + "model.layers.6.mlp.up_proj.weight": "model_1.safetensors", + "model.layers.6.mlp.down_proj.weight": "model_1.safetensors", + "model.layers.6.experts.gate_up_proj": "model_1.safetensors", + "model.layers.6.experts.down_proj": "model_1.safetensors", + "model.layers.6.layer_scalar": "model_1.safetensors", + "model.layers.7.input_layernorm.weight": "model_1.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model_1.safetensors", + "model.layers.7.post_feedforward_layernorm.weight": "model_1.safetensors", + "model.layers.7.self_attn.q_norm.weight": "model_1.safetensors", + "model.layers.7.self_attn.k_norm.weight": "model_1.safetensors", + "model.layers.7.pre_feedforward_layernorm.weight": "model_1.safetensors", + "model.layers.7.post_feedforward_layernorm_1.weight": "model_1.safetensors", + "model.layers.7.router.scale": "model_1.safetensors", + "model.layers.7.router.per_expert_scale": "model_1.safetensors", + "model.layers.7.pre_feedforward_layernorm_2.weight": "model_1.safetensors", + "model.layers.7.post_feedforward_layernorm_2.weight": "model_1.safetensors", + "model.layers.7.router.proj.weight": "model_1.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model_1.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model_1.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model_1.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model_1.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model_1.safetensors", + "model.layers.7.mlp.up_proj.weight": "model_1.safetensors", + "model.layers.7.mlp.down_proj.weight": "model_1.safetensors", + "model.layers.7.experts.gate_up_proj": "model_1.safetensors", + "model.layers.7.experts.down_proj": "model_1.safetensors", + "model.layers.7.layer_scalar": "model_1.safetensors", + "model.layers.8.input_layernorm.weight": "model_1.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model_1.safetensors", + "model.layers.8.post_feedforward_layernorm.weight": "model_1.safetensors", + "model.layers.8.self_attn.q_norm.weight": "model_1.safetensors", + "model.layers.8.self_attn.k_norm.weight": "model_1.safetensors", + "model.layers.8.pre_feedforward_layernorm.weight": "model_1.safetensors", + "model.layers.8.post_feedforward_layernorm_1.weight": "model_1.safetensors", + "model.layers.8.router.scale": "model_1.safetensors", + "model.layers.8.router.per_expert_scale": "model_1.safetensors", + "model.layers.8.pre_feedforward_layernorm_2.weight": "model_1.safetensors", + "model.layers.8.post_feedforward_layernorm_2.weight": "model_1.safetensors", + "model.layers.8.router.proj.weight": "model_1.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model_1.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model_1.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model_1.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model_1.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model_1.safetensors", + "model.layers.8.mlp.up_proj.weight": "model_1.safetensors", + "model.layers.8.mlp.down_proj.weight": "model_1.safetensors", + "model.layers.8.experts.gate_up_proj": "model_1.safetensors", + "model.layers.8.experts.down_proj": "model_1.safetensors", + "model.layers.8.layer_scalar": "model_1.safetensors", + "model.layers.9.input_layernorm.weight": "model_1.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model_1.safetensors", + "model.layers.9.post_feedforward_layernorm.weight": "model_1.safetensors", + "model.layers.9.self_attn.q_norm.weight": "model_1.safetensors", + "model.layers.9.self_attn.k_norm.weight": "model_1.safetensors", + "model.layers.9.pre_feedforward_layernorm.weight": "model_1.safetensors", + "model.layers.9.post_feedforward_layernorm_1.weight": "model_1.safetensors", + "model.layers.9.router.scale": "model_1.safetensors", + "model.layers.9.router.per_expert_scale": "model_1.safetensors", + "model.layers.9.pre_feedforward_layernorm_2.weight": "model_1.safetensors", + "model.layers.9.post_feedforward_layernorm_2.weight": "model_1.safetensors", + "model.layers.9.router.proj.weight": "model_1.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model_1.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model_1.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model_1.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model_1.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model_1.safetensors", + "model.layers.9.mlp.up_proj.weight": "model_1.safetensors", + "model.layers.9.mlp.down_proj.weight": "model_1.safetensors", + "model.layers.9.experts.gate_up_proj": "model_1.safetensors", + "model.layers.9.experts.down_proj": "model_1.safetensors", + "model.layers.9.layer_scalar": "model_2.safetensors", + "model.layers.10.input_layernorm.weight": "model_2.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model_2.safetensors", + "model.layers.10.post_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.10.self_attn.q_norm.weight": "model_2.safetensors", + "model.layers.10.self_attn.k_norm.weight": "model_2.safetensors", + "model.layers.10.pre_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.10.post_feedforward_layernorm_1.weight": "model_2.safetensors", + "model.layers.10.router.scale": "model_2.safetensors", + "model.layers.10.router.per_expert_scale": "model_2.safetensors", + "model.layers.10.pre_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.10.post_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.10.router.proj.weight": "model_2.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model_2.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model_2.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model_2.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model_2.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model_2.safetensors", + "model.layers.10.mlp.up_proj.weight": "model_2.safetensors", + "model.layers.10.mlp.down_proj.weight": "model_2.safetensors", + "model.layers.10.experts.gate_up_proj": "model_2.safetensors", + "model.layers.10.experts.down_proj": "model_2.safetensors", + "model.layers.10.layer_scalar": "model_2.safetensors", + "model.layers.11.input_layernorm.weight": "model_2.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model_2.safetensors", + "model.layers.11.post_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.11.self_attn.q_norm.weight": "model_2.safetensors", + "model.layers.11.self_attn.k_norm.weight": "model_2.safetensors", + "model.layers.11.pre_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.11.post_feedforward_layernorm_1.weight": "model_2.safetensors", + "model.layers.11.router.scale": "model_2.safetensors", + "model.layers.11.router.per_expert_scale": "model_2.safetensors", + "model.layers.11.pre_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.11.post_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.11.router.proj.weight": "model_2.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model_2.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model_2.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model_2.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model_2.safetensors", + "model.layers.11.mlp.up_proj.weight": "model_2.safetensors", + "model.layers.11.mlp.down_proj.weight": "model_2.safetensors", + "model.layers.11.experts.gate_up_proj": "model_2.safetensors", + "model.layers.11.experts.down_proj": "model_2.safetensors", + "model.layers.11.layer_scalar": "model_2.safetensors", + "model.layers.12.input_layernorm.weight": "model_2.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model_2.safetensors", + "model.layers.12.post_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.12.self_attn.q_norm.weight": "model_2.safetensors", + "model.layers.12.self_attn.k_norm.weight": "model_2.safetensors", + "model.layers.12.pre_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.12.post_feedforward_layernorm_1.weight": "model_2.safetensors", + "model.layers.12.router.scale": "model_2.safetensors", + "model.layers.12.router.per_expert_scale": "model_2.safetensors", + "model.layers.12.pre_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.12.post_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.12.router.proj.weight": "model_2.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model_2.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model_2.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model_2.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model_2.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model_2.safetensors", + "model.layers.12.mlp.up_proj.weight": "model_2.safetensors", + "model.layers.12.mlp.down_proj.weight": "model_2.safetensors", + "model.layers.12.experts.gate_up_proj": "model_2.safetensors", + "model.layers.12.experts.down_proj": "model_2.safetensors", + "model.layers.12.layer_scalar": "model_2.safetensors", + "model.layers.13.input_layernorm.weight": "model_2.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model_2.safetensors", + "model.layers.13.post_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.13.self_attn.q_norm.weight": "model_2.safetensors", + "model.layers.13.self_attn.k_norm.weight": "model_2.safetensors", + "model.layers.13.pre_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.13.post_feedforward_layernorm_1.weight": "model_2.safetensors", + "model.layers.13.router.scale": "model_2.safetensors", + "model.layers.13.router.per_expert_scale": "model_2.safetensors", + "model.layers.13.pre_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.13.post_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.13.router.proj.weight": "model_2.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model_2.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model_2.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model_2.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model_2.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model_2.safetensors", + "model.layers.13.mlp.up_proj.weight": "model_2.safetensors", + "model.layers.13.mlp.down_proj.weight": "model_2.safetensors", + "model.layers.13.experts.gate_up_proj": "model_2.safetensors", + "model.layers.13.experts.down_proj": "model_2.safetensors", + "model.layers.13.layer_scalar": "model_2.safetensors", + "model.layers.14.input_layernorm.weight": "model_2.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model_2.safetensors", + "model.layers.14.post_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.14.self_attn.q_norm.weight": "model_2.safetensors", + "model.layers.14.self_attn.k_norm.weight": "model_2.safetensors", + "model.layers.14.pre_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.14.post_feedforward_layernorm_1.weight": "model_2.safetensors", + "model.layers.14.router.scale": "model_2.safetensors", + "model.layers.14.router.per_expert_scale": "model_2.safetensors", + "model.layers.14.pre_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.14.post_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.14.router.proj.weight": "model_2.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model_2.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model_2.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model_2.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model_2.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model_2.safetensors", + "model.layers.14.mlp.up_proj.weight": "model_2.safetensors", + "model.layers.14.mlp.down_proj.weight": "model_2.safetensors", + "model.layers.14.experts.gate_up_proj": "model_2.safetensors", + "model.layers.14.experts.down_proj": "model_2.safetensors", + "model.layers.14.layer_scalar": "model_2.safetensors", + "model.layers.15.input_layernorm.weight": "model_2.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model_2.safetensors", + "model.layers.15.post_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.15.self_attn.q_norm.weight": "model_2.safetensors", + "model.layers.15.self_attn.k_norm.weight": "model_2.safetensors", + "model.layers.15.pre_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.15.post_feedforward_layernorm_1.weight": "model_2.safetensors", + "model.layers.15.router.scale": "model_2.safetensors", + "model.layers.15.router.per_expert_scale": "model_2.safetensors", + "model.layers.15.pre_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.15.post_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.15.router.proj.weight": "model_2.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model_2.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model_2.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model_2.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model_2.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model_2.safetensors", + "model.layers.15.mlp.up_proj.weight": "model_2.safetensors", + "model.layers.15.mlp.down_proj.weight": "model_2.safetensors", + "model.layers.15.experts.gate_up_proj": "model_2.safetensors", + "model.layers.15.experts.down_proj": "model_3.safetensors", + "model.layers.15.layer_scalar": "model_3.safetensors", + "model.layers.16.input_layernorm.weight": "model_3.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model_3.safetensors", + "model.layers.16.post_feedforward_layernorm.weight": "model_3.safetensors", + "model.layers.16.self_attn.q_norm.weight": "model_3.safetensors", + "model.layers.16.self_attn.k_norm.weight": "model_3.safetensors", + "model.layers.16.pre_feedforward_layernorm.weight": "model_3.safetensors", + "model.layers.16.post_feedforward_layernorm_1.weight": "model_3.safetensors", + "model.layers.16.router.scale": "model_3.safetensors", + "model.layers.16.router.per_expert_scale": "model_3.safetensors", + "model.layers.16.pre_feedforward_layernorm_2.weight": "model_3.safetensors", + "model.layers.16.post_feedforward_layernorm_2.weight": "model_3.safetensors", + "model.layers.16.router.proj.weight": "model_3.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model_3.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model_3.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model_3.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model_3.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model_3.safetensors", + "model.layers.16.mlp.up_proj.weight": "model_3.safetensors", + "model.layers.16.mlp.down_proj.weight": "model_3.safetensors", + "model.layers.16.experts.gate_up_proj": "model_3.safetensors", + "model.layers.16.experts.down_proj": "model_3.safetensors", + "model.layers.16.layer_scalar": "model_3.safetensors", + "model.layers.17.input_layernorm.weight": "model_3.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model_3.safetensors", + "model.layers.17.post_feedforward_layernorm.weight": "model_3.safetensors", + "model.layers.17.self_attn.q_norm.weight": "model_3.safetensors", + "model.layers.17.self_attn.k_norm.weight": "model_3.safetensors", + "model.layers.17.pre_feedforward_layernorm.weight": "model_3.safetensors", + "model.layers.17.post_feedforward_layernorm_1.weight": "model_3.safetensors", + "model.layers.17.router.scale": "model_3.safetensors", + "model.layers.17.router.per_expert_scale": "model_3.safetensors", + "model.layers.17.pre_feedforward_layernorm_2.weight": "model_3.safetensors", + "model.layers.17.post_feedforward_layernorm_2.weight": "model_3.safetensors", + "model.layers.17.router.proj.weight": "model_3.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model_3.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model_3.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model_3.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model_3.safetensors", + "model.layers.17.mlp.up_proj.weight": "model_3.safetensors", + "model.layers.17.mlp.down_proj.weight": "model_3.safetensors", + "model.layers.17.experts.gate_up_proj": "model_3.safetensors", + "model.layers.17.experts.down_proj": "model_3.safetensors", + "model.layers.17.layer_scalar": "model_3.safetensors", + "model.layers.18.input_layernorm.weight": "model_3.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model_3.safetensors", + "model.layers.18.post_feedforward_layernorm.weight": "model_3.safetensors", + "model.layers.18.self_attn.q_norm.weight": "model_3.safetensors", + "model.layers.18.self_attn.k_norm.weight": "model_3.safetensors", + "model.layers.18.pre_feedforward_layernorm.weight": "model_3.safetensors", + "model.layers.18.post_feedforward_layernorm_1.weight": "model_3.safetensors", + "model.layers.18.router.scale": "model_3.safetensors", + "model.layers.18.router.per_expert_scale": "model_3.safetensors", + "model.layers.18.pre_feedforward_layernorm_2.weight": "model_3.safetensors", + "model.layers.18.post_feedforward_layernorm_2.weight": "model_3.safetensors", + "model.layers.18.router.proj.weight": "model_3.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model_3.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model_3.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model_3.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model_3.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model_3.safetensors", + "model.layers.18.mlp.up_proj.weight": "model_3.safetensors", + "model.layers.18.mlp.down_proj.weight": "model_3.safetensors", + "model.layers.18.experts.gate_up_proj": "model_3.safetensors", + "model.layers.18.experts.down_proj": "model_3.safetensors", + "model.layers.18.layer_scalar": "model_3.safetensors", + "model.layers.19.input_layernorm.weight": "model_3.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model_3.safetensors", + "model.layers.19.post_feedforward_layernorm.weight": "model_3.safetensors", + "model.layers.19.self_attn.q_norm.weight": "model_3.safetensors", + "model.layers.19.self_attn.k_norm.weight": "model_3.safetensors", + "model.layers.19.pre_feedforward_layernorm.weight": "model_3.safetensors", + "model.layers.19.post_feedforward_layernorm_1.weight": "model_3.safetensors", + "model.layers.19.router.scale": "model_3.safetensors", + "model.layers.19.router.per_expert_scale": "model_3.safetensors", + "model.layers.19.pre_feedforward_layernorm_2.weight": "model_3.safetensors", + "model.layers.19.post_feedforward_layernorm_2.weight": "model_3.safetensors", + "model.layers.19.router.proj.weight": "model_3.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model_3.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model_3.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model_3.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model_3.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model_3.safetensors", + "model.layers.19.mlp.up_proj.weight": "model_3.safetensors", + "model.layers.19.mlp.down_proj.weight": "model_3.safetensors", + "model.layers.19.experts.gate_up_proj": "model_3.safetensors", + "model.layers.19.experts.down_proj": "model_3.safetensors", + "model.layers.19.layer_scalar": "model_3.safetensors", + "model.layers.20.input_layernorm.weight": "model_3.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model_3.safetensors", + "model.layers.20.post_feedforward_layernorm.weight": "model_3.safetensors", + "model.layers.20.self_attn.q_norm.weight": "model_3.safetensors", + "model.layers.20.self_attn.k_norm.weight": "model_3.safetensors", + "model.layers.20.pre_feedforward_layernorm.weight": "model_3.safetensors", + "model.layers.20.post_feedforward_layernorm_1.weight": "model_3.safetensors", + "model.layers.20.router.scale": "model_3.safetensors", + "model.layers.20.router.per_expert_scale": "model_3.safetensors", + "model.layers.20.pre_feedforward_layernorm_2.weight": "model_3.safetensors", + "model.layers.20.post_feedforward_layernorm_2.weight": "model_3.safetensors", + "model.layers.20.router.proj.weight": "model_3.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model_3.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model_3.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model_3.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model_3.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model_3.safetensors", + "model.layers.20.mlp.up_proj.weight": "model_3.safetensors", + "model.layers.20.mlp.down_proj.weight": "model_3.safetensors", + "model.layers.20.experts.gate_up_proj": "model_3.safetensors", + "model.layers.20.experts.down_proj": "model_3.safetensors", + "model.layers.20.layer_scalar": "model_4.safetensors", + "model.layers.21.input_layernorm.weight": "model_4.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model_4.safetensors", + "model.layers.21.post_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.21.self_attn.q_norm.weight": "model_4.safetensors", + "model.layers.21.self_attn.k_norm.weight": "model_4.safetensors", + "model.layers.21.pre_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.21.post_feedforward_layernorm_1.weight": "model_4.safetensors", + "model.layers.21.router.scale": "model_4.safetensors", + "model.layers.21.router.per_expert_scale": "model_4.safetensors", + "model.layers.21.pre_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.21.post_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.21.router.proj.weight": "model_4.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model_4.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model_4.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model_4.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model_4.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model_4.safetensors", + "model.layers.21.mlp.up_proj.weight": "model_4.safetensors", + "model.layers.21.mlp.down_proj.weight": "model_4.safetensors", + "model.layers.21.experts.gate_up_proj": "model_4.safetensors", + "model.layers.21.experts.down_proj": "model_4.safetensors", + "model.layers.21.layer_scalar": "model_4.safetensors", + "model.layers.22.input_layernorm.weight": "model_4.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model_4.safetensors", + "model.layers.22.post_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.22.self_attn.q_norm.weight": "model_4.safetensors", + "model.layers.22.self_attn.k_norm.weight": "model_4.safetensors", + "model.layers.22.pre_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.22.post_feedforward_layernorm_1.weight": "model_4.safetensors", + "model.layers.22.router.scale": "model_4.safetensors", + "model.layers.22.router.per_expert_scale": "model_4.safetensors", + "model.layers.22.pre_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.22.post_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.22.router.proj.weight": "model_4.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model_4.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model_4.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model_4.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model_4.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model_4.safetensors", + "model.layers.22.mlp.up_proj.weight": "model_4.safetensors", + "model.layers.22.mlp.down_proj.weight": "model_4.safetensors", + "model.layers.22.experts.gate_up_proj": "model_4.safetensors", + "model.layers.22.experts.down_proj": "model_4.safetensors", + "model.layers.22.layer_scalar": "model_4.safetensors", + "model.layers.23.input_layernorm.weight": "model_4.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model_4.safetensors", + "model.layers.23.post_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.23.self_attn.q_norm.weight": "model_4.safetensors", + "model.layers.23.self_attn.k_norm.weight": "model_4.safetensors", + "model.layers.23.pre_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.23.post_feedforward_layernorm_1.weight": "model_4.safetensors", + "model.layers.23.router.scale": "model_4.safetensors", + "model.layers.23.router.per_expert_scale": "model_4.safetensors", + "model.layers.23.pre_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.23.post_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.23.router.proj.weight": "model_4.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model_4.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model_4.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model_4.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model_4.safetensors", + "model.layers.23.mlp.up_proj.weight": "model_4.safetensors", + "model.layers.23.mlp.down_proj.weight": "model_4.safetensors", + "model.layers.23.experts.gate_up_proj": "model_4.safetensors", + "model.layers.23.experts.down_proj": "model_4.safetensors", + "model.layers.23.layer_scalar": "model_4.safetensors", + "model.layers.24.input_layernorm.weight": "model_4.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model_4.safetensors", + "model.layers.24.post_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.24.self_attn.q_norm.weight": "model_4.safetensors", + "model.layers.24.self_attn.k_norm.weight": "model_4.safetensors", + "model.layers.24.pre_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.24.post_feedforward_layernorm_1.weight": "model_4.safetensors", + "model.layers.24.router.scale": "model_4.safetensors", + "model.layers.24.router.per_expert_scale": "model_4.safetensors", + "model.layers.24.pre_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.24.post_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.24.router.proj.weight": "model_4.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model_4.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model_4.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model_4.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model_4.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model_4.safetensors", + "model.layers.24.mlp.up_proj.weight": "model_4.safetensors", + "model.layers.24.mlp.down_proj.weight": "model_4.safetensors", + "model.layers.24.experts.gate_up_proj": "model_4.safetensors", + "model.layers.24.experts.down_proj": "model_4.safetensors", + "model.layers.24.layer_scalar": "model_4.safetensors", + "model.layers.25.input_layernorm.weight": "model_4.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model_4.safetensors", + "model.layers.25.post_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.25.self_attn.q_norm.weight": "model_4.safetensors", + "model.layers.25.self_attn.k_norm.weight": "model_4.safetensors", + "model.layers.25.pre_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.25.post_feedforward_layernorm_1.weight": "model_4.safetensors", + "model.layers.25.router.scale": "model_4.safetensors", + "model.layers.25.router.per_expert_scale": "model_4.safetensors", + "model.layers.25.pre_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.25.post_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.25.router.proj.weight": "model_4.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model_4.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model_4.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model_4.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model_4.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model_4.safetensors", + "model.layers.25.mlp.up_proj.weight": "model_4.safetensors", + "model.layers.25.mlp.down_proj.weight": "model_4.safetensors", + "model.layers.25.experts.gate_up_proj": "model_4.safetensors", + "model.layers.25.experts.down_proj": "model_4.safetensors", + "model.layers.25.layer_scalar": "model_4.safetensors", + "model.layers.26.input_layernorm.weight": "model_4.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model_4.safetensors", + "model.layers.26.post_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.26.self_attn.q_norm.weight": "model_4.safetensors", + "model.layers.26.self_attn.k_norm.weight": "model_4.safetensors", + "model.layers.26.pre_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.26.post_feedforward_layernorm_1.weight": "model_4.safetensors", + "model.layers.26.router.scale": "model_4.safetensors", + "model.layers.26.router.per_expert_scale": "model_4.safetensors", + "model.layers.26.pre_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.26.post_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.26.router.proj.weight": "model_4.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model_4.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model_4.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model_4.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model_4.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model_4.safetensors", + "model.layers.26.mlp.up_proj.weight": "model_4.safetensors", + "model.layers.26.mlp.down_proj.weight": "model_4.safetensors", + "model.layers.26.experts.gate_up_proj": "model_4.safetensors", + "model.layers.26.experts.down_proj": "model_5.safetensors", + "model.layers.26.layer_scalar": "model_5.safetensors", + "model.layers.27.input_layernorm.weight": "model_5.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model_5.safetensors", + "model.layers.27.post_feedforward_layernorm.weight": "model_5.safetensors", + "model.layers.27.self_attn.q_norm.weight": "model_5.safetensors", + "model.layers.27.self_attn.k_norm.weight": "model_5.safetensors", + "model.layers.27.pre_feedforward_layernorm.weight": "model_5.safetensors", + "model.layers.27.post_feedforward_layernorm_1.weight": "model_5.safetensors", + "model.layers.27.router.scale": "model_5.safetensors", + "model.layers.27.router.per_expert_scale": "model_5.safetensors", + "model.layers.27.pre_feedforward_layernorm_2.weight": "model_5.safetensors", + "model.layers.27.post_feedforward_layernorm_2.weight": "model_5.safetensors", + "model.layers.27.router.proj.weight": "model_5.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model_5.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model_5.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model_5.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model_5.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model_5.safetensors", + "model.layers.27.mlp.up_proj.weight": "model_5.safetensors", + "model.layers.27.mlp.down_proj.weight": "model_5.safetensors", + "model.layers.27.experts.gate_up_proj": "model_5.safetensors", + "model.layers.27.experts.down_proj": "model_5.safetensors", + "model.layers.27.layer_scalar": "model_5.safetensors", + "model.layers.28.input_layernorm.weight": "model_5.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model_5.safetensors", + "model.layers.28.post_feedforward_layernorm.weight": "model_5.safetensors", + "model.layers.28.self_attn.q_norm.weight": "model_5.safetensors", + "model.layers.28.self_attn.k_norm.weight": "model_5.safetensors", + "model.layers.28.pre_feedforward_layernorm.weight": "model_5.safetensors", + "model.layers.28.post_feedforward_layernorm_1.weight": "model_5.safetensors", + "model.layers.28.router.scale": "model_5.safetensors", + "model.layers.28.router.per_expert_scale": "model_5.safetensors", + "model.layers.28.pre_feedforward_layernorm_2.weight": "model_5.safetensors", + "model.layers.28.post_feedforward_layernorm_2.weight": "model_5.safetensors", + "model.layers.28.router.proj.weight": "model_5.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model_5.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model_5.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model_5.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model_5.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model_5.safetensors", + "model.layers.28.mlp.up_proj.weight": "model_5.safetensors", + "model.layers.28.mlp.down_proj.weight": "model_5.safetensors", + "model.layers.28.experts.gate_up_proj": "model_5.safetensors", + "model.layers.28.experts.down_proj": "model_5.safetensors", + "model.layers.28.layer_scalar": "model_5.safetensors", + "model.layers.29.input_layernorm.weight": "model_5.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model_5.safetensors", + "model.layers.29.post_feedforward_layernorm.weight": "model_5.safetensors", + "model.layers.29.self_attn.q_norm.weight": "model_5.safetensors", + "model.layers.29.self_attn.k_norm.weight": "model_5.safetensors", + "model.layers.29.pre_feedforward_layernorm.weight": "model_5.safetensors", + "model.layers.29.post_feedforward_layernorm_1.weight": "model_5.safetensors", + "model.layers.29.router.scale": "model_5.safetensors", + "model.layers.29.router.per_expert_scale": "model_5.safetensors", + "model.layers.29.pre_feedforward_layernorm_2.weight": "model_5.safetensors", + "model.layers.29.post_feedforward_layernorm_2.weight": "model_5.safetensors", + "model.layers.29.router.proj.weight": "model_5.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model_5.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model_5.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model_5.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model_5.safetensors", + "model.layers.29.mlp.up_proj.weight": "model_5.safetensors", + "model.layers.29.mlp.down_proj.weight": "model_5.safetensors", + "model.layers.29.experts.gate_up_proj": "model_5.safetensors", + "model.layers.29.experts.down_proj": "model_5.safetensors", + "model.layers.29.layer_scalar": "model_5.safetensors", + "model.norm.weight": "model_5.safetensors" + } +} \ No newline at end of file diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/210/model_0.safetensors b/26_09_08_sft_email_v4_v3/export/gemma4/210/model_0.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1b84869c1d61d3a4427982abe8f2bd1969d8ca78 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/210/model_0.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:235b37f041fa036c111cda5de1c386e4acd1c02c72ec049d14cd8ec9a51ed8c8 +size 18219689400 diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/210/model_1.safetensors b/26_09_08_sft_email_v4_v3/export/gemma4/210/model_1.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..db6ecf30fa51d9d9a1c3ee24c71884e09911835e --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/210/model_1.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53988b57f3f6885db1404438af6643c77807f3ab246fd8f5c103c21680780433 +size 17354616348 diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/210/model_2.safetensors b/26_09_08_sft_email_v4_v3/export/gemma4/210/model_2.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..938f421c94b16d7534f21121e80c06298bae6e14 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/210/model_2.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d409016595f4d1672722b49f995b09f73292e334c58d715980dbd34d9cc3c047 +size 18580955536 diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/210/model_3.safetensors b/26_09_08_sft_email_v4_v3/export/gemma4/210/model_3.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..acb6c6b180ccbc46b60c6218f0b1c4978a0de9a8 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/210/model_3.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:583d9bbfd8fc19ea1936117ae1a651e6469205767e71b6a4a65b8d65eedef1f1 +size 17354616460 diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/210/model_4.safetensors b/26_09_08_sft_email_v4_v3/export/gemma4/210/model_4.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4593ba11720feed4743490f5bcc28c5e129b63c1 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/210/model_4.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b686f62b6a8ff945889290ab68dfcfdfd7156822fc5fd86f32c79c75f3f28789 +size 18580955512 diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/210/model_5.safetensors b/26_09_08_sft_email_v4_v3/export/gemma4/210/model_5.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7ce8a36f71a2c37101c5d7e7f041ab2da7896f42 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/210/model_5.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdb49e2609c628513f93b5f57ef56edcd487b6256f66a902509231b24da3fdfd +size 10841862960 diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/210/ok b/26_09_08_sft_email_v4_v3/export/gemma4/210/ok new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/210/tokenizer.json b/26_09_08_sft_email_v4_v3/export/gemma4/210/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..1ff9f3e3439a939b971f9919e821bf87e835a503 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/210/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc8d3a0ce36466ccc1278bf987df5f71db1719b9ca6b4118264f45cb627bfe0f +size 32169626 diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/210/tokenizer_config.json b/26_09_08_sft_email_v4_v3/export/gemma4/210/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..375b25dc8be85705251e41be1c25310d24932051 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/210/tokenizer_config.json @@ -0,0 +1,74 @@ +{ + "audio_token": "<|audio|>", + "backend": "tokenizers", + "boa_token": "<|audio>", + "boi_token": "<|image>", + "bos_token": "", + "eoa_token": "", + "eoc_token": "", + "eoi_token": "", + "eos_token": "", + "eot_token": "", + "escape_token": "<|\"|>", + "etc_token": "", + "etd_token": "", + "etr_token": "", + "extra_special_tokens": [ + "<|video|>" + ], + "image_token": "<|image|>", + "mask_token": "", + "model_max_length": 1000000000000000019884624838656, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma4Processor", + "response_schema": { + "type": "object", + "properties": { + "role": { + "const": "assistant" + }, + "thinking": { + "type": "string" + }, + "content": { + "type": "string" + }, + "tool_calls": { + "x-regex-iterator": "<\\|tool_call>(.*?)", + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "const": "function" + }, + "function": { + "type": "object", + "x-regex": "call\\:(?P\\w+)(?P\\{.*\\})", + "properties": { + "name": { + "type": "string" + }, + "arguments": { + "type": "object", + "x-parser": "gemma4-tool-call", + "additionalProperties": {} + } + } + } + } + } + } + }, + "x-regex": "(\\<\\|channel\\>thought\\n(?P.*?)\\)?(?P\\<\\|tool_call\\>.*\\)?(?P(?:(?!\\)(?!\\<\\|tool_response\\>).)+)?(?:\\|\\<\\|tool_response\\>)?" + }, + "soc_token": "<|channel>", + "sot_token": "<|turn>", + "stc_token": "<|tool_call>", + "std_token": "<|tool>", + "str_token": "<|tool_response>", + "think_token": "<|think|>", + "tokenizer_class": "GemmaTokenizer", + "unk_token": "" +} diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/70/chat_template.jinja b/26_09_08_sft_email_v4_v3/export/gemma4/70/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..20e0028946060065b95e9376890a07f52b3c7b6f --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/70/chat_template.jinja @@ -0,0 +1,390 @@ +{# + Template: Google Gemma 4 Canonical Chat Template + Author: Google Gemma Engineering Team + Published: 2026-07-09 + Context: Fixed tool-calling loops, turn closures, and thinking content-ordering. +#} +{%- macro format_parameters(properties, required, filter_keys=false) -%} + {%- set standard_keys = ['properties', 'required', 'nullable'] -%} + {%- set ns = namespace(found_first=false) -%} + {%- for key, value in properties | dictsort -%} + {%- set add_comma = false -%} + {%- if not filter_keys or key not in standard_keys -%} + {%- if ns.found_first %},{% endif -%} + {%- set ns.found_first = true -%} + {{ key }}:{ + {%- if value['description'] -%} + description:<|"|>{{ value['description'] }}<|"|> + {%- set add_comma = true -%} + {%- endif -%} + {%- if value['type'] | upper == 'STRING' -%} + {%- if value['enum'] -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + enum:{{ format_argument(value['enum']) }} + {%- endif -%} + {%- elif value['type'] | upper == 'ARRAY' -%} + {%- if value['items'] is mapping and value['items'] -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + items:{ + {%- set ns_items = namespace(found_first=false) -%} + {%- for item_key, item_value in value['items'] | dictsort -%} + {%- if item_value is not none -%} + {%- if ns_items.found_first %},{% endif -%} + {%- set ns_items.found_first = true -%} + {%- if item_key == 'properties' -%} + properties:{ + {%- if item_value is mapping -%} + {{- format_parameters(item_value, value['items']['required'] | default([])) -}} + {%- endif -%} + } + {%- elif item_key == 'required' -%} + required:[ + {%- for req_item in item_value -%} + <|"|>{{- req_item -}}<|"|> + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + ] + {%- elif item_key == 'type' -%} + {%- if item_value is string -%} + type:{{ format_argument(item_value | upper) }} + {%- else -%} + type:{{ format_argument(item_value | map('upper') | list) }} + {%- endif -%} + {%- else -%} + {{ item_key }}:{{ format_argument(item_value) }} + {%- endif -%} + {%- endif -%} + {%- endfor -%} + } + {%- endif -%} + {%- endif -%} + {%- if value['nullable'] %} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + nullable:true + {%- endif -%} + {%- if value['type'] | upper == 'OBJECT' -%} + {%- if value['properties'] is defined and value['properties'] is mapping -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + properties:{ + {{- format_parameters(value['properties'], value['required'] | default([])) -}} + } + {%- elif value is mapping -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + properties:{ + {{- format_parameters(value, value['required'] | default([]), filter_keys=true) -}} + } + {%- endif -%} + {%- if value['required'] -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + required:[ + {%- for item in value['required'] | default([]) -%} + <|"|>{{- item -}}<|"|> + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + ] + {%- endif -%} + {%- endif -%} + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} + type:<|"|>{{ value['type'] | upper }}<|"|>} + {%- endif -%} + {%- endfor -%} +{%- endmacro -%} +{%- macro format_function_declaration(tool_data) -%} + declaration:{{- tool_data['function']['name'] -}}{description:<|"|>{{- tool_data['function']['description'] -}}<|"|> + {%- set params = tool_data['function']['parameters'] -%} + {%- if params -%} + ,parameters:{ + {%- if params['properties'] -%} + properties:{ {{- format_parameters(params['properties'], params['required']) -}} }, + {%- endif -%} + {%- if params['required'] -%} + required:[ + {%- for item in params['required'] -%} + <|"|>{{- item -}}<|"|> + {{- ',' if not loop.last -}} + {%- endfor -%} + ], + {%- endif -%} + {%- if params['type'] -%} + type:<|"|>{{- params['type'] | upper -}}<|"|>} + {%- endif -%} + {%- endif -%} + {%- if 'response' in tool_data['function'] -%} + {%- set response_declaration = tool_data['function']['response'] -%} + ,response:{ + {%- if response_declaration['description'] -%} + description:<|"|>{{- response_declaration['description'] -}}<|"|>, + {%- endif -%} + {%- if response_declaration['type'] | upper == 'OBJECT' -%} + type:<|"|>{{- response_declaration['type'] | upper -}}<|"|>} + {%- endif -%} + {%- endif -%} + } +{%- endmacro -%} +{%- macro format_argument(argument, escape_keys=True) -%} + {%- if argument is none -%} + {{- 'null' -}} + {%- elif argument is string -%} + {{- '<|"|>' + argument + '<|"|>' -}} + {%- elif argument is boolean -%} + {{- 'true' if argument else 'false' -}} + {%- elif argument is mapping -%} + {{- '{' -}} + {%- set ns = namespace(found_first=false) -%} + {%- for key, value in argument | dictsort -%} + {%- if ns.found_first %},{% endif -%} + {%- set ns.found_first = true -%} + {%- if escape_keys -%} + {{- '<|"|>' + key + '<|"|>' -}} + {%- else -%} + {{- key -}} + {%- endif -%} + :{{- format_argument(value, escape_keys=escape_keys) -}} + {%- endfor -%} + {{- '}' -}} + {%- elif argument is sequence -%} + {{- '[' -}} + {%- for item in argument -%} + {{- format_argument(item, escape_keys=escape_keys) -}} + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + {{- ']' -}} + {%- else -%} + {{- argument -}} + {%- endif -%} +{%- endmacro -%} +{%- macro strip_thinking(text) -%} + {%- set ns = namespace(result='') -%} + {%- for part in text.split('') -%} + {%- if '<|channel>' in part -%} + {%- set ns.result = ns.result + part.split('<|channel>')[0] -%} + {%- else -%} + {%- set ns.result = ns.result + part -%} + {%- endif -%} + {%- endfor -%} + {{- ns.result | trim -}} +{%- endmacro -%} + +{%- macro format_tool_response_block(tool_name, response) -%} + {{- '<|tool_response>' -}} + {%- if response is mapping -%} + {{- 'response:' + tool_name + '{' -}} + {%- for key, value in response | dictsort -%} + {{- key -}}:{{- format_argument(value, escape_keys=False) -}} + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + {{- '}' -}} + {%- else -%} + {{- 'response:' + tool_name + '{value:' + format_argument(response, escape_keys=False) + '}' -}} + {%- endif -%} + {{- '' -}} +{%- endmacro -%} + +{#- ===== SETUP ===== -#} +{%- set ns = namespace(prev_message_type=None, prev_non_tool_role=None) -%} +{%- set loop_messages = messages -%} +{%- set enable_thinking = enable_thinking | default(false) -%} +{%- set preserve_thinking = preserve_thinking | default(false) -%} +{{- bos_token -}} +{#- Handle System/Tool Definitions Block -#} +{%- if enable_thinking or tools or (messages and messages[0]['role'] in ['system', 'developer']) -%} + {{- '<|turn>system\n' -}} + {#- Inject Thinking token at the very top of the FIRST system turn -#} + {%- if enable_thinking -%} + {{- '<|think|>\n' -}} + {%- set ns.prev_message_type = 'think' -%} + {%- endif -%} + {%- if messages and messages[0]['role'] in ['system', 'developer'] -%} + {%- if messages[0]['content'] is string -%} + {{- messages[0]['content'] | trim -}} + {%- elif messages[0]['content'] is sequence -%} + {%- for item in messages[0]['content'] -%} + {{- item['text'] | trim + ' '-}} + {%- endfor -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} + {%- endif -%} + {%- if tools -%} + {%- for tool in tools %} + {{- '<|tool>' -}} + {{- format_function_declaration(tool) | trim -}} + {{- '' -}} + {%- endfor %} + {%- set ns.prev_message_type = 'tool' -%} + {%- endif -%} + {{- '\n' -}} +{%- endif %} + +{#- Pre-scan: find last user message index for reasoning guard -#} +{%- set ns_turn = namespace(last_user_idx=-1) -%} +{%- for i in range(loop_messages | length) -%} + {%- if loop_messages[i]['role'] == 'user' -%} + {%- set ns_turn.last_user_idx = i -%} + {%- endif -%} +{%- endfor -%} + +{#- Loop through messages -#} +{%- for message in loop_messages -%} + {%- if message['role'] != 'tool' -%} + {%- set ns.prev_message_type = None -%} + {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%} + {#- Detect continuation using tracked state — O(1) instead of O(n) backward scan -#} + {%- set continue_same_model_turn = (role == 'model' and ns.prev_non_tool_role == 'assistant') -%} + {%- if not continue_same_model_turn -%} + {{- '<|turn>' + role + '\n' }} + + {%- endif -%} + + {#- Render reasoning/reasoning_content as thinking channel -#} + {%- set thinking_text = message.get('reasoning') or message.get('reasoning_content') -%} + {%- set thinking_gate = (loop.index0 > ns_turn.last_user_idx) or (preserve_thinking and message.get('tool_calls')) -%} + {%- if thinking_text and thinking_gate -%} + {{- '<|channel>thought\n' + thinking_text + '\n' -}} + {%- endif -%} + + {%- if message.get('tool_calls') -%} + {%- for tool_call in message.get('tool_calls') -%} + {%- set function = tool_call['function'] -%} + {{- '<|tool_call>call:' + function['name'] + '{' -}} + {%- if function['arguments'] is mapping -%} + {%- set ns_args = namespace(found_first=false) -%} + {%- for key, value in function['arguments'] | dictsort -%} + {%- if ns_args.found_first %},{% endif -%} + {%- set ns_args.found_first = true -%} + {{- key -}}:{{- format_argument(value, escape_keys=False) -}} + {%- endfor -%} + {%- elif function['arguments'] is none -%} + {%- else -%} + {{- raise_exception( + "chat_template: tool_calls[].function.arguments must be a " + "JSON object (mapping), not a string. Deserialize arguments " + "before passing to the template." + ) -}} + {%- endif -%} + {{- '}' -}} + {%- endfor -%} + {%- set ns.prev_message_type = 'tool_call' -%} + {%- endif -%} + + {%- set ns_tr_out = namespace(flag=false) -%} + {%- if message.get('tool_responses') -%} + {#- Legacy: tool_responses embedded on the assistant message (Google/Gemma native) -#} + {%- for tool_response in message.get('tool_responses') -%} + {{- format_tool_response_block(tool_response['name'] | default('unknown', true), tool_response['response']) -}} + {%- set ns_tr_out.flag = true -%} + {%- set ns.prev_message_type = 'tool_response' -%} + {%- endfor -%} + {%- elif message.get('tool_calls') -%} + {#- OpenAI Chat Completions: forward-scan consecutive role:tool messages -#} + {%- set ns_tool_scan = namespace(stopped=false) -%} + {%- for k in range(loop.index0 + 1, loop_messages | length) -%} + {%- if ns_tool_scan.stopped -%} + {%- elif loop_messages[k]['role'] != 'tool' -%} + {%- set ns_tool_scan.stopped = true -%} + {%- else -%} + {%- set follow = loop_messages[k] -%} + {#- Resolve tool_call_id to function name -#} + {%- set ns_tname = namespace(name=follow.get('name') or 'unknown') -%} + {%- for tc in message.get('tool_calls') -%} + {%- if tc.get('id') == follow.get('tool_call_id') -%} + {%- set ns_tname.name = tc['function']['name'] -%} + {%- endif -%} + {%- endfor -%} + {#- Handle content as string or content-parts array -#} + {%- set tool_body = follow.get('content') -%} + {%- if tool_body is string -%} + {{- format_tool_response_block(ns_tname.name, tool_body) -}} + {%- elif tool_body is sequence and tool_body is not string -%} + {%- set ns_txt = namespace(s='') -%} + {%- for part in tool_body -%} + {%- if part.get('type') == 'text' -%} + {%- set ns_txt.s = ns_txt.s + (part.get('text') | default('')) -%} + {%- endif -%} + {%- endfor -%} + {{- format_tool_response_block(ns_tname.name, ns_txt.s) -}} + {%- for part in tool_body -%} + {%- if part.get('type') in ['image', 'image_url'] -%} + {{- '<|image|>' -}} + {%- elif part.get('type') in ['audio', 'input_audio'] -%} + {{- '<|audio|>' -}} + {%- elif part.get('type') == 'video' -%} + {{- '<|video|>' -}} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{- format_tool_response_block(ns_tname.name, tool_body) -}} + {%- endif -%} + {%- set ns_tr_out.flag = true -%} + {%- set ns.prev_message_type = 'tool_response' -%} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + + {%- set captured_content -%} + {%- if message.get('content') is string -%} + {%- if role == 'model' -%} + {{- strip_thinking(message['content']) -}} + {%- else -%} + {{- message['content'] | trim -}} + {%- endif -%} + {%- elif message.get('content') is sequence -%} + {%- for item in message['content'] -%} + {%- if item.get('type') == 'text' -%} + {%- if role == 'model' -%} + {{- strip_thinking(item['text']) -}} + {%- else -%} + {{- item['text'] | trim -}} + {%- endif -%} + {%- elif item.get('type') in ['image', 'image_url'] -%} + {{- '<|image|>' -}} + {%- elif item.get('type') in ['audio', 'input_audio'] -%} + {{- '<|audio|>' -}} + {%- elif item.get('type') == 'video' -%} + {{- '<|video|>' -}} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + {%- endset -%} + + {{- captured_content -}} + {%- set has_content = captured_content | trim | length > 0 -%} + + {#- Forward-scan: find next non-tool message role for continuation detection -#} + {%- set next_nt = namespace(role=None, found=false) -%} + {%- for j in range(loop.index0 + 1, loop_messages | length) -%} + {%- if not next_nt.found -%} + {%- if loop_messages[j]['role'] != 'tool' -%} + {%- set next_nt.role = loop_messages[j]['role'] -%} + {%- set next_nt.found = true -%} + {%- endif -%} + {%- endif -%} + {%- endfor -%} + + {%- set continues_into_next = ( + role == 'model' + and next_nt.role == 'assistant' + and (not message.get('tool_calls') or ns_tr_out.flag) + ) -%} + + {%- if ns.prev_message_type == 'tool_call' and not ns_tr_out.flag -%} + {{- '<|tool_response>' -}} + {%- elif continues_into_next -%} + {%- elif not (ns_tr_out.flag and not has_content and not next_nt.found) -%} + {{- '\n' -}} + {%- endif -%} + + {#- Track previous non-tool role for next iteration (avoids O(n) backward scan) -#} + {%- set ns.prev_non_tool_role = message['role'] -%} + {%- endif -%} +{%- endfor -%} + +{%- if add_generation_prompt -%} + {%- if ns.prev_message_type != 'tool_response' and ns.prev_message_type != 'tool_call' -%} + {{- '<|turn>model\n' -}} + {%- if not enable_thinking -%} + {{- '<|channel>thought\n' -}} + {%- endif -%} + {%- elif ns.prev_message_type == 'tool_response' and enable_thinking -%} + {{- '<|channel>thought\n' -}} + {%- endif -%} +{%- endif -%} diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/70/config.json b/26_09_08_sft_email_v4_v3/export/gemma4/70/config.json new file mode 100644 index 0000000000000000000000000000000000000000..518607d204ad0bf72d56220356c72e1516d05041 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/70/config.json @@ -0,0 +1,83 @@ +{ + "attention_bias": false, + "attention_dropout": 0.0, + "attention_k_eq_v": true, + "bos_token_id": 2, + "dtype": "bfloat16", + "enable_moe_block": true, + "eos_token_id": 1, + "final_logit_softcapping": 30.0, + "global_head_dim": 512, + "head_dim": 256, + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 2816, + "hidden_size_per_layer_input": 0, + "initializer_range": 0.02, + "intermediate_size": 2112, + "layer_types": [ + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 262144, + "model_type": "gemma4_text", + "moe_intermediate_size": 704, + "num_attention_heads": 16, + "num_experts": 128, + "num_global_key_value_heads": 2, + "num_hidden_layers": 30, + "num_key_value_heads": 8, + "num_kv_shared_layers": 0, + "pad_token_id": 0, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "full_attention": { + "partial_rotary_factor": 0.25, + "rope_theta": 1000000.0, + "rope_type": "proportional" + }, + "sliding_attention": { + "rope_theta": 10000.0, + "rope_type": "default" + } + }, + "sliding_window": 1024, + "tie_word_embeddings": true, + "top_k_experts": 8, + "use_bidirectional_attention": "vision", + "use_cache": true, + "use_double_wide_mlp": false, + "vocab_size": 262144, + "vocab_size_per_layer_input": 262144, + "architectures": [ + "Gemma4ForCausalLM" + ], + "transformers_version": "5.5.0.dev0" +} \ No newline at end of file diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/70/generation_config.json b/26_09_08_sft_email_v4_v3/export/gemma4/70/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e605bb4523b1462ea9d9a3810b9e3ecf7ab7b1f6 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/70/generation_config.json @@ -0,0 +1,14 @@ +{ + "bos_token_id": 2, + "do_sample": true, + "eos_token_id": [ + 1, + 106, + 50 + ], + "pad_token_id": 0, + "temperature": 1.0, + "top_k": 64, + "top_p": 0.95, + "transformers_version": "5.5.0.dev0" +} diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/70/model.safetensors.index.json b/26_09_08_sft_email_v4_v3/export/gemma4/70/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..c71a599d7645570620b963357bbc89a0b4af188b --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/70/model.safetensors.index.json @@ -0,0 +1,1053 @@ +{ + "metadata": { + "fast_llm_metadata": { + "fast_llm_version": "0.4.0", + "model": "gpt", + "format": "gemma4", + "config": { + "type": "gpt", + "base_model": { + "decoder": { + "type": "pattern", + "blocks": { + "full_attention": { + "type": "decoder", + "mixer": { + "type": "attention", + "rotary": { + "type": "proportional", + "theta": 1000000.0, + "partial_rotary_factor": 0.25 + }, + "heads": 16, + "head_groups": 2, + "head_size": 512, + "add_linear_biases": false, + "dropout": 0.0, + "softmax_scale_power": 0.0, + "implementation": "sdpa", + "query_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "key_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "value_norm": { + "type": "fixed_rms_norm", + "epsilon": 1e-06 + }, + "shared_key_value": true + }, + "mlp": { + "type": "hybrid_moe", + "dense": { + "type": "mlp", + "pre_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "post_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "intermediate_size": 2112, + "add_linear_biases": false, + "gated": true, + "activation": "gelu", + "recompute_level": "full" + }, + "routed": { + "type": "moe", + "pre_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "post_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "intermediate_size": 704, + "add_linear_biases": false, + "gated": true, + "activation": "gelu", + "recompute_level": "full", + "router_normalization": { + "type": "fixed_rms_norm", + "epsilon": 1e-06 + }, + "router_scale": { + "enabled": true + }, + "router_input_scale": 0.018844459036110227, + "router_per_expert_scale": { + "enabled": true + }, + "experts": 128, + "experts_per_token": 8, + "auxiliary_loss_coefficient": 0.0 + } + }, + "normalization": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "pre_mlp_normalization": { + "type": "none" + }, + "post_mixer_normalization": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "post_mlp_normalization": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "output_scale": { + "lr_scale": 0.0, + "enabled": true + } + }, + "sliding_attention": { + "type": "decoder", + "mixer": { + "type": "attention", + "rotary": { + "type": "default", + "theta": 10000.0 + }, + "heads": 16, + "head_groups": 8, + "head_size": 256, + "add_linear_biases": false, + "dropout": 0.0, + "window_size": 1024, + "softmax_scale_power": 0.0, + "implementation": "flash", + "query_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "key_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "value_norm": { + "type": "fixed_rms_norm", + "epsilon": 1e-06 + } + }, + "mlp": { + "type": "hybrid_moe", + "dense": { + "type": "mlp", + "pre_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "post_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "intermediate_size": 2112, + "add_linear_biases": false, + "gated": true, + "activation": "gelu", + "recompute_level": "full" + }, + "routed": { + "type": "moe", + "pre_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "post_norm": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "intermediate_size": 704, + "add_linear_biases": false, + "gated": true, + "activation": "gelu", + "recompute_level": "full", + "router_normalization": { + "type": "fixed_rms_norm", + "epsilon": 1e-06 + }, + "router_scale": { + "enabled": true + }, + "router_input_scale": 0.018844459036110227, + "router_per_expert_scale": { + "enabled": true + }, + "experts": 128, + "experts_per_token": 8, + "auxiliary_loss_coefficient": 0.0 + } + }, + "normalization": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "pre_mlp_normalization": { + "type": "none" + }, + "post_mixer_normalization": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "post_mlp_normalization": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "output_scale": { + "lr_scale": 0.0, + "enabled": true + } + } + }, + "pattern": [ + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention" + ], + "num_blocks": 30 + }, + "embeddings": { + "vocab_size": 262144, + "vocab_parallel": false, + "embedding_scale": 53.0659966456864 + }, + "head": { + "normalization": { + "type": "rms_norm", + "epsilon": 1e-06 + }, + "cross_entropy_splits": 8, + "final_logit_softcap": 30.0 + }, + "tied_embedding_weight": true, + "hidden_size": 2816, + "peft": { + "type": "none" + } + }, + "multi_stage": { + "zero_stage": 3 + }, + "distributed": { + "tensor_parallel": 2, + "sequence_tensor_parallel": true, + "sequence_data_parallel": 4, + "world_size": 32, + "rank": 0, + "local_world_size": 8, + "timeout": 3600.0, + "compute_dtype": "bfloat16" + } + }, + "shards": [ + "weights" + ], + "metadata": { + "optimizer": { + "current_step": 70, + "grad_scaler": { + "type": "NoopGradScaler" + } + }, + "completed_steps": 70, + "metrics": { + "training": { + "batch_size": 2097152, + "load_balancing_loss": 823.45361328125, + "lm_head_loss": 0.15507733821868896, + "total_steps": 210, + "completed_steps": 70, + "consumed_tokens": 146800640, + "percent_done": 33.333333333333336, + "step_time_ms": 27782.785279676318, + "step_time_average_ms": 28139.88637443898, + "remaining_time": 3939.584092421457, + "completion_time": 1788872635.9465992, + "skipped_iters": 0, + "nan_iters": 0, + "model_tflops": 60428.45211557096, + "hardware_tflops": 74970.05489154597, + "tokens_per_sec_per_gpu": 2358.870766205753, + "run": 0, + "grad_norm": 4.336630344390869, + "learning_rate": 1.5611870653623826e-05, + "loss_scale": 1.0, + "reserved": 61408.0, + "allocated": 21478.49755859375, + "max_allocated": 54787.2607421875, + "max_reserved": 61408.0, + "global_max_reserved": 61408.0 + } + } + } + }, + "model_config": { + "vocab_size": 262144, + "num_attention_heads": 16, + "num_key_value_heads": 8, + "head_dim": 256, + "global_head_dim": 512, + "num_global_key_value_heads": 2, + "attention_bias": false, + "attention_dropout": 0.0, + "sliding_window": 1024, + "rms_norm_eps": 1e-06, + "attention_k_eq_v": true, + "rope_parameters": { + "sliding_attention": { + "rope_type": "default", + "rope_theta": 10000.0 + }, + "full_attention": { + "rope_type": "proportional", + "rope_theta": 1000000.0, + "partial_rotary_factor": 0.25 + } + }, + "intermediate_size": 2112, + "hidden_activation": "gelu_pytorch_tanh", + "num_experts": 128, + "top_k_experts": 8, + "moe_intermediate_size": 704, + "enable_moe_block": true, + "num_hidden_layers": 30, + "layer_types": [ + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention" + ], + "final_logit_softcapping": 30.0, + "tie_word_embeddings": true, + "hidden_size": 2816, + "hidden_size_per_layer_input": 0, + "use_bidirectional_attention": null, + "model_type": "gemma4_text", + "architectures": [ + "Gemma4ForCausalLM" + ] + }, + "format": "pt" + }, + "weight_map": { + "model.embed_tokens.weight": "model_0.safetensors", + "model.layers.0.input_layernorm.weight": "model_0.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model_0.safetensors", + "model.layers.0.post_feedforward_layernorm.weight": "model_0.safetensors", + "model.layers.0.self_attn.q_norm.weight": "model_0.safetensors", + "model.layers.0.self_attn.k_norm.weight": "model_0.safetensors", + "model.layers.0.pre_feedforward_layernorm.weight": "model_0.safetensors", + "model.layers.0.post_feedforward_layernorm_1.weight": "model_0.safetensors", + "model.layers.0.router.scale": "model_0.safetensors", + "model.layers.0.router.per_expert_scale": "model_0.safetensors", + "model.layers.0.pre_feedforward_layernorm_2.weight": "model_0.safetensors", + "model.layers.0.post_feedforward_layernorm_2.weight": "model_0.safetensors", + "model.layers.0.router.proj.weight": "model_0.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model_0.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model_0.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model_0.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model_0.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model_0.safetensors", + "model.layers.0.mlp.up_proj.weight": "model_0.safetensors", + "model.layers.0.mlp.down_proj.weight": "model_0.safetensors", + "model.layers.0.experts.gate_up_proj": "model_0.safetensors", + "model.layers.0.experts.down_proj": "model_0.safetensors", + "model.layers.0.layer_scalar": "model_0.safetensors", + "model.layers.1.input_layernorm.weight": "model_0.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model_0.safetensors", + "model.layers.1.post_feedforward_layernorm.weight": "model_0.safetensors", + "model.layers.1.self_attn.q_norm.weight": "model_0.safetensors", + "model.layers.1.self_attn.k_norm.weight": "model_0.safetensors", + "model.layers.1.pre_feedforward_layernorm.weight": "model_0.safetensors", + "model.layers.1.post_feedforward_layernorm_1.weight": "model_0.safetensors", + "model.layers.1.router.scale": "model_0.safetensors", + "model.layers.1.router.per_expert_scale": "model_0.safetensors", + "model.layers.1.pre_feedforward_layernorm_2.weight": "model_0.safetensors", + "model.layers.1.post_feedforward_layernorm_2.weight": "model_0.safetensors", + "model.layers.1.router.proj.weight": "model_0.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model_0.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model_0.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model_0.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model_0.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model_0.safetensors", + "model.layers.1.mlp.up_proj.weight": "model_0.safetensors", + "model.layers.1.mlp.down_proj.weight": "model_0.safetensors", + "model.layers.1.experts.gate_up_proj": "model_0.safetensors", + "model.layers.1.experts.down_proj": "model_0.safetensors", + "model.layers.1.layer_scalar": "model_0.safetensors", + "model.layers.2.input_layernorm.weight": "model_0.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model_0.safetensors", + "model.layers.2.post_feedforward_layernorm.weight": "model_0.safetensors", + "model.layers.2.self_attn.q_norm.weight": "model_0.safetensors", + "model.layers.2.self_attn.k_norm.weight": "model_0.safetensors", + "model.layers.2.pre_feedforward_layernorm.weight": "model_0.safetensors", + "model.layers.2.post_feedforward_layernorm_1.weight": "model_0.safetensors", + "model.layers.2.router.scale": "model_0.safetensors", + "model.layers.2.router.per_expert_scale": "model_0.safetensors", + "model.layers.2.pre_feedforward_layernorm_2.weight": "model_0.safetensors", + "model.layers.2.post_feedforward_layernorm_2.weight": "model_0.safetensors", + "model.layers.2.router.proj.weight": "model_0.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model_0.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model_0.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model_0.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model_0.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model_0.safetensors", + "model.layers.2.mlp.up_proj.weight": "model_0.safetensors", + "model.layers.2.mlp.down_proj.weight": "model_0.safetensors", + "model.layers.2.experts.gate_up_proj": "model_0.safetensors", + "model.layers.2.experts.down_proj": "model_0.safetensors", + "model.layers.2.layer_scalar": "model_0.safetensors", + "model.layers.3.input_layernorm.weight": "model_0.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model_0.safetensors", + "model.layers.3.post_feedforward_layernorm.weight": "model_0.safetensors", + "model.layers.3.self_attn.q_norm.weight": "model_0.safetensors", + "model.layers.3.self_attn.k_norm.weight": "model_0.safetensors", + "model.layers.3.pre_feedforward_layernorm.weight": "model_0.safetensors", + "model.layers.3.post_feedforward_layernorm_1.weight": "model_0.safetensors", + "model.layers.3.router.scale": "model_0.safetensors", + "model.layers.3.router.per_expert_scale": "model_0.safetensors", + "model.layers.3.pre_feedforward_layernorm_2.weight": "model_0.safetensors", + "model.layers.3.post_feedforward_layernorm_2.weight": "model_0.safetensors", + "model.layers.3.router.proj.weight": "model_0.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model_0.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model_0.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model_0.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model_0.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model_0.safetensors", + "model.layers.3.mlp.up_proj.weight": "model_0.safetensors", + "model.layers.3.mlp.down_proj.weight": "model_0.safetensors", + "model.layers.3.experts.gate_up_proj": "model_0.safetensors", + "model.layers.3.experts.down_proj": "model_0.safetensors", + "model.layers.3.layer_scalar": "model_0.safetensors", + "model.layers.4.input_layernorm.weight": "model_0.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model_0.safetensors", + "model.layers.4.post_feedforward_layernorm.weight": "model_0.safetensors", + "model.layers.4.self_attn.q_norm.weight": "model_0.safetensors", + "model.layers.4.self_attn.k_norm.weight": "model_0.safetensors", + "model.layers.4.pre_feedforward_layernorm.weight": "model_0.safetensors", + "model.layers.4.post_feedforward_layernorm_1.weight": "model_0.safetensors", + "model.layers.4.router.scale": "model_0.safetensors", + "model.layers.4.router.per_expert_scale": "model_0.safetensors", + "model.layers.4.pre_feedforward_layernorm_2.weight": "model_0.safetensors", + "model.layers.4.post_feedforward_layernorm_2.weight": "model_0.safetensors", + "model.layers.4.router.proj.weight": "model_0.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model_0.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model_0.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model_0.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model_0.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model_0.safetensors", + "model.layers.4.mlp.up_proj.weight": "model_0.safetensors", + "model.layers.4.mlp.down_proj.weight": "model_0.safetensors", + "model.layers.4.experts.gate_up_proj": "model_0.safetensors", + "model.layers.4.experts.down_proj": "model_1.safetensors", + "model.layers.4.layer_scalar": "model_1.safetensors", + "model.layers.5.input_layernorm.weight": "model_1.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model_1.safetensors", + "model.layers.5.post_feedforward_layernorm.weight": "model_1.safetensors", + "model.layers.5.self_attn.q_norm.weight": "model_1.safetensors", + "model.layers.5.self_attn.k_norm.weight": "model_1.safetensors", + "model.layers.5.pre_feedforward_layernorm.weight": "model_1.safetensors", + "model.layers.5.post_feedforward_layernorm_1.weight": "model_1.safetensors", + "model.layers.5.router.scale": "model_1.safetensors", + "model.layers.5.router.per_expert_scale": "model_1.safetensors", + "model.layers.5.pre_feedforward_layernorm_2.weight": "model_1.safetensors", + "model.layers.5.post_feedforward_layernorm_2.weight": "model_1.safetensors", + "model.layers.5.router.proj.weight": "model_1.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model_1.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model_1.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model_1.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model_1.safetensors", + "model.layers.5.mlp.up_proj.weight": "model_1.safetensors", + "model.layers.5.mlp.down_proj.weight": "model_1.safetensors", + "model.layers.5.experts.gate_up_proj": "model_1.safetensors", + "model.layers.5.experts.down_proj": "model_1.safetensors", + "model.layers.5.layer_scalar": "model_1.safetensors", + "model.layers.6.input_layernorm.weight": "model_1.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model_1.safetensors", + "model.layers.6.post_feedforward_layernorm.weight": "model_1.safetensors", + "model.layers.6.self_attn.q_norm.weight": "model_1.safetensors", + "model.layers.6.self_attn.k_norm.weight": "model_1.safetensors", + "model.layers.6.pre_feedforward_layernorm.weight": "model_1.safetensors", + "model.layers.6.post_feedforward_layernorm_1.weight": "model_1.safetensors", + "model.layers.6.router.scale": "model_1.safetensors", + "model.layers.6.router.per_expert_scale": "model_1.safetensors", + "model.layers.6.pre_feedforward_layernorm_2.weight": "model_1.safetensors", + "model.layers.6.post_feedforward_layernorm_2.weight": "model_1.safetensors", + "model.layers.6.router.proj.weight": "model_1.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model_1.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model_1.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model_1.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model_1.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model_1.safetensors", + "model.layers.6.mlp.up_proj.weight": "model_1.safetensors", + "model.layers.6.mlp.down_proj.weight": "model_1.safetensors", + "model.layers.6.experts.gate_up_proj": "model_1.safetensors", + "model.layers.6.experts.down_proj": "model_1.safetensors", + "model.layers.6.layer_scalar": "model_1.safetensors", + "model.layers.7.input_layernorm.weight": "model_1.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model_1.safetensors", + "model.layers.7.post_feedforward_layernorm.weight": "model_1.safetensors", + "model.layers.7.self_attn.q_norm.weight": "model_1.safetensors", + "model.layers.7.self_attn.k_norm.weight": "model_1.safetensors", + "model.layers.7.pre_feedforward_layernorm.weight": "model_1.safetensors", + "model.layers.7.post_feedforward_layernorm_1.weight": "model_1.safetensors", + "model.layers.7.router.scale": "model_1.safetensors", + "model.layers.7.router.per_expert_scale": "model_1.safetensors", + "model.layers.7.pre_feedforward_layernorm_2.weight": "model_1.safetensors", + "model.layers.7.post_feedforward_layernorm_2.weight": "model_1.safetensors", + "model.layers.7.router.proj.weight": "model_1.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model_1.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model_1.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model_1.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model_1.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model_1.safetensors", + "model.layers.7.mlp.up_proj.weight": "model_1.safetensors", + "model.layers.7.mlp.down_proj.weight": "model_1.safetensors", + "model.layers.7.experts.gate_up_proj": "model_1.safetensors", + "model.layers.7.experts.down_proj": "model_1.safetensors", + "model.layers.7.layer_scalar": "model_1.safetensors", + "model.layers.8.input_layernorm.weight": "model_1.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model_1.safetensors", + "model.layers.8.post_feedforward_layernorm.weight": "model_1.safetensors", + "model.layers.8.self_attn.q_norm.weight": "model_1.safetensors", + "model.layers.8.self_attn.k_norm.weight": "model_1.safetensors", + "model.layers.8.pre_feedforward_layernorm.weight": "model_1.safetensors", + "model.layers.8.post_feedforward_layernorm_1.weight": "model_1.safetensors", + "model.layers.8.router.scale": "model_1.safetensors", + "model.layers.8.router.per_expert_scale": "model_1.safetensors", + "model.layers.8.pre_feedforward_layernorm_2.weight": "model_1.safetensors", + "model.layers.8.post_feedforward_layernorm_2.weight": "model_1.safetensors", + "model.layers.8.router.proj.weight": "model_1.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model_1.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model_1.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model_1.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model_1.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model_1.safetensors", + "model.layers.8.mlp.up_proj.weight": "model_1.safetensors", + "model.layers.8.mlp.down_proj.weight": "model_1.safetensors", + "model.layers.8.experts.gate_up_proj": "model_1.safetensors", + "model.layers.8.experts.down_proj": "model_1.safetensors", + "model.layers.8.layer_scalar": "model_1.safetensors", + "model.layers.9.input_layernorm.weight": "model_1.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model_1.safetensors", + "model.layers.9.post_feedforward_layernorm.weight": "model_1.safetensors", + "model.layers.9.self_attn.q_norm.weight": "model_1.safetensors", + "model.layers.9.self_attn.k_norm.weight": "model_1.safetensors", + "model.layers.9.pre_feedforward_layernorm.weight": "model_1.safetensors", + "model.layers.9.post_feedforward_layernorm_1.weight": "model_1.safetensors", + "model.layers.9.router.scale": "model_1.safetensors", + "model.layers.9.router.per_expert_scale": "model_1.safetensors", + "model.layers.9.pre_feedforward_layernorm_2.weight": "model_1.safetensors", + "model.layers.9.post_feedforward_layernorm_2.weight": "model_1.safetensors", + "model.layers.9.router.proj.weight": "model_1.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model_1.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model_1.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model_1.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model_1.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model_1.safetensors", + "model.layers.9.mlp.up_proj.weight": "model_1.safetensors", + "model.layers.9.mlp.down_proj.weight": "model_1.safetensors", + "model.layers.9.experts.gate_up_proj": "model_1.safetensors", + "model.layers.9.experts.down_proj": "model_1.safetensors", + "model.layers.9.layer_scalar": "model_2.safetensors", + "model.layers.10.input_layernorm.weight": "model_2.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model_2.safetensors", + "model.layers.10.post_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.10.self_attn.q_norm.weight": "model_2.safetensors", + "model.layers.10.self_attn.k_norm.weight": "model_2.safetensors", + "model.layers.10.pre_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.10.post_feedforward_layernorm_1.weight": "model_2.safetensors", + "model.layers.10.router.scale": "model_2.safetensors", + "model.layers.10.router.per_expert_scale": "model_2.safetensors", + "model.layers.10.pre_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.10.post_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.10.router.proj.weight": "model_2.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model_2.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model_2.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model_2.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model_2.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model_2.safetensors", + "model.layers.10.mlp.up_proj.weight": "model_2.safetensors", + "model.layers.10.mlp.down_proj.weight": "model_2.safetensors", + "model.layers.10.experts.gate_up_proj": "model_2.safetensors", + "model.layers.10.experts.down_proj": "model_2.safetensors", + "model.layers.10.layer_scalar": "model_2.safetensors", + "model.layers.11.input_layernorm.weight": "model_2.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model_2.safetensors", + "model.layers.11.post_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.11.self_attn.q_norm.weight": "model_2.safetensors", + "model.layers.11.self_attn.k_norm.weight": "model_2.safetensors", + "model.layers.11.pre_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.11.post_feedforward_layernorm_1.weight": "model_2.safetensors", + "model.layers.11.router.scale": "model_2.safetensors", + "model.layers.11.router.per_expert_scale": "model_2.safetensors", + "model.layers.11.pre_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.11.post_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.11.router.proj.weight": "model_2.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model_2.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model_2.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model_2.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model_2.safetensors", + "model.layers.11.mlp.up_proj.weight": "model_2.safetensors", + "model.layers.11.mlp.down_proj.weight": "model_2.safetensors", + "model.layers.11.experts.gate_up_proj": "model_2.safetensors", + "model.layers.11.experts.down_proj": "model_2.safetensors", + "model.layers.11.layer_scalar": "model_2.safetensors", + "model.layers.12.input_layernorm.weight": "model_2.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model_2.safetensors", + "model.layers.12.post_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.12.self_attn.q_norm.weight": "model_2.safetensors", + "model.layers.12.self_attn.k_norm.weight": "model_2.safetensors", + "model.layers.12.pre_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.12.post_feedforward_layernorm_1.weight": "model_2.safetensors", + "model.layers.12.router.scale": "model_2.safetensors", + "model.layers.12.router.per_expert_scale": "model_2.safetensors", + "model.layers.12.pre_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.12.post_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.12.router.proj.weight": "model_2.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model_2.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model_2.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model_2.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model_2.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model_2.safetensors", + "model.layers.12.mlp.up_proj.weight": "model_2.safetensors", + "model.layers.12.mlp.down_proj.weight": "model_2.safetensors", + "model.layers.12.experts.gate_up_proj": "model_2.safetensors", + "model.layers.12.experts.down_proj": "model_2.safetensors", + "model.layers.12.layer_scalar": "model_2.safetensors", + "model.layers.13.input_layernorm.weight": "model_2.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model_2.safetensors", + "model.layers.13.post_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.13.self_attn.q_norm.weight": "model_2.safetensors", + "model.layers.13.self_attn.k_norm.weight": "model_2.safetensors", + "model.layers.13.pre_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.13.post_feedforward_layernorm_1.weight": "model_2.safetensors", + "model.layers.13.router.scale": "model_2.safetensors", + "model.layers.13.router.per_expert_scale": "model_2.safetensors", + "model.layers.13.pre_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.13.post_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.13.router.proj.weight": "model_2.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model_2.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model_2.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model_2.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model_2.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model_2.safetensors", + "model.layers.13.mlp.up_proj.weight": "model_2.safetensors", + "model.layers.13.mlp.down_proj.weight": "model_2.safetensors", + "model.layers.13.experts.gate_up_proj": "model_2.safetensors", + "model.layers.13.experts.down_proj": "model_2.safetensors", + "model.layers.13.layer_scalar": "model_2.safetensors", + "model.layers.14.input_layernorm.weight": "model_2.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model_2.safetensors", + "model.layers.14.post_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.14.self_attn.q_norm.weight": "model_2.safetensors", + "model.layers.14.self_attn.k_norm.weight": "model_2.safetensors", + "model.layers.14.pre_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.14.post_feedforward_layernorm_1.weight": "model_2.safetensors", + "model.layers.14.router.scale": "model_2.safetensors", + "model.layers.14.router.per_expert_scale": "model_2.safetensors", + "model.layers.14.pre_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.14.post_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.14.router.proj.weight": "model_2.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model_2.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model_2.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model_2.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model_2.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model_2.safetensors", + "model.layers.14.mlp.up_proj.weight": "model_2.safetensors", + "model.layers.14.mlp.down_proj.weight": "model_2.safetensors", + "model.layers.14.experts.gate_up_proj": "model_2.safetensors", + "model.layers.14.experts.down_proj": "model_2.safetensors", + "model.layers.14.layer_scalar": "model_2.safetensors", + "model.layers.15.input_layernorm.weight": "model_2.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model_2.safetensors", + "model.layers.15.post_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.15.self_attn.q_norm.weight": "model_2.safetensors", + "model.layers.15.self_attn.k_norm.weight": "model_2.safetensors", + "model.layers.15.pre_feedforward_layernorm.weight": "model_2.safetensors", + "model.layers.15.post_feedforward_layernorm_1.weight": "model_2.safetensors", + "model.layers.15.router.scale": "model_2.safetensors", + "model.layers.15.router.per_expert_scale": "model_2.safetensors", + "model.layers.15.pre_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.15.post_feedforward_layernorm_2.weight": "model_2.safetensors", + "model.layers.15.router.proj.weight": "model_2.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model_2.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model_2.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model_2.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model_2.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model_2.safetensors", + "model.layers.15.mlp.up_proj.weight": "model_2.safetensors", + "model.layers.15.mlp.down_proj.weight": "model_2.safetensors", + "model.layers.15.experts.gate_up_proj": "model_2.safetensors", + "model.layers.15.experts.down_proj": "model_3.safetensors", + "model.layers.15.layer_scalar": "model_3.safetensors", + "model.layers.16.input_layernorm.weight": "model_3.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model_3.safetensors", + "model.layers.16.post_feedforward_layernorm.weight": "model_3.safetensors", + "model.layers.16.self_attn.q_norm.weight": "model_3.safetensors", + "model.layers.16.self_attn.k_norm.weight": "model_3.safetensors", + "model.layers.16.pre_feedforward_layernorm.weight": "model_3.safetensors", + "model.layers.16.post_feedforward_layernorm_1.weight": "model_3.safetensors", + "model.layers.16.router.scale": "model_3.safetensors", + "model.layers.16.router.per_expert_scale": "model_3.safetensors", + "model.layers.16.pre_feedforward_layernorm_2.weight": "model_3.safetensors", + "model.layers.16.post_feedforward_layernorm_2.weight": "model_3.safetensors", + "model.layers.16.router.proj.weight": "model_3.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model_3.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model_3.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model_3.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model_3.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model_3.safetensors", + "model.layers.16.mlp.up_proj.weight": "model_3.safetensors", + "model.layers.16.mlp.down_proj.weight": "model_3.safetensors", + "model.layers.16.experts.gate_up_proj": "model_3.safetensors", + "model.layers.16.experts.down_proj": "model_3.safetensors", + "model.layers.16.layer_scalar": "model_3.safetensors", + "model.layers.17.input_layernorm.weight": "model_3.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model_3.safetensors", + "model.layers.17.post_feedforward_layernorm.weight": "model_3.safetensors", + "model.layers.17.self_attn.q_norm.weight": "model_3.safetensors", + "model.layers.17.self_attn.k_norm.weight": "model_3.safetensors", + "model.layers.17.pre_feedforward_layernorm.weight": "model_3.safetensors", + "model.layers.17.post_feedforward_layernorm_1.weight": "model_3.safetensors", + "model.layers.17.router.scale": "model_3.safetensors", + "model.layers.17.router.per_expert_scale": "model_3.safetensors", + "model.layers.17.pre_feedforward_layernorm_2.weight": "model_3.safetensors", + "model.layers.17.post_feedforward_layernorm_2.weight": "model_3.safetensors", + "model.layers.17.router.proj.weight": "model_3.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model_3.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model_3.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model_3.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model_3.safetensors", + "model.layers.17.mlp.up_proj.weight": "model_3.safetensors", + "model.layers.17.mlp.down_proj.weight": "model_3.safetensors", + "model.layers.17.experts.gate_up_proj": "model_3.safetensors", + "model.layers.17.experts.down_proj": "model_3.safetensors", + "model.layers.17.layer_scalar": "model_3.safetensors", + "model.layers.18.input_layernorm.weight": "model_3.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model_3.safetensors", + "model.layers.18.post_feedforward_layernorm.weight": "model_3.safetensors", + "model.layers.18.self_attn.q_norm.weight": "model_3.safetensors", + "model.layers.18.self_attn.k_norm.weight": "model_3.safetensors", + "model.layers.18.pre_feedforward_layernorm.weight": "model_3.safetensors", + "model.layers.18.post_feedforward_layernorm_1.weight": "model_3.safetensors", + "model.layers.18.router.scale": "model_3.safetensors", + "model.layers.18.router.per_expert_scale": "model_3.safetensors", + "model.layers.18.pre_feedforward_layernorm_2.weight": "model_3.safetensors", + "model.layers.18.post_feedforward_layernorm_2.weight": "model_3.safetensors", + "model.layers.18.router.proj.weight": "model_3.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model_3.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model_3.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model_3.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model_3.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model_3.safetensors", + "model.layers.18.mlp.up_proj.weight": "model_3.safetensors", + "model.layers.18.mlp.down_proj.weight": "model_3.safetensors", + "model.layers.18.experts.gate_up_proj": "model_3.safetensors", + "model.layers.18.experts.down_proj": "model_3.safetensors", + "model.layers.18.layer_scalar": "model_3.safetensors", + "model.layers.19.input_layernorm.weight": "model_3.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model_3.safetensors", + "model.layers.19.post_feedforward_layernorm.weight": "model_3.safetensors", + "model.layers.19.self_attn.q_norm.weight": "model_3.safetensors", + "model.layers.19.self_attn.k_norm.weight": "model_3.safetensors", + "model.layers.19.pre_feedforward_layernorm.weight": "model_3.safetensors", + "model.layers.19.post_feedforward_layernorm_1.weight": "model_3.safetensors", + "model.layers.19.router.scale": "model_3.safetensors", + "model.layers.19.router.per_expert_scale": "model_3.safetensors", + "model.layers.19.pre_feedforward_layernorm_2.weight": "model_3.safetensors", + "model.layers.19.post_feedforward_layernorm_2.weight": "model_3.safetensors", + "model.layers.19.router.proj.weight": "model_3.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model_3.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model_3.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model_3.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model_3.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model_3.safetensors", + "model.layers.19.mlp.up_proj.weight": "model_3.safetensors", + "model.layers.19.mlp.down_proj.weight": "model_3.safetensors", + "model.layers.19.experts.gate_up_proj": "model_3.safetensors", + "model.layers.19.experts.down_proj": "model_3.safetensors", + "model.layers.19.layer_scalar": "model_3.safetensors", + "model.layers.20.input_layernorm.weight": "model_3.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model_3.safetensors", + "model.layers.20.post_feedforward_layernorm.weight": "model_3.safetensors", + "model.layers.20.self_attn.q_norm.weight": "model_3.safetensors", + "model.layers.20.self_attn.k_norm.weight": "model_3.safetensors", + "model.layers.20.pre_feedforward_layernorm.weight": "model_3.safetensors", + "model.layers.20.post_feedforward_layernorm_1.weight": "model_3.safetensors", + "model.layers.20.router.scale": "model_3.safetensors", + "model.layers.20.router.per_expert_scale": "model_3.safetensors", + "model.layers.20.pre_feedforward_layernorm_2.weight": "model_3.safetensors", + "model.layers.20.post_feedforward_layernorm_2.weight": "model_3.safetensors", + "model.layers.20.router.proj.weight": "model_3.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model_3.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model_3.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model_3.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model_3.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model_3.safetensors", + "model.layers.20.mlp.up_proj.weight": "model_3.safetensors", + "model.layers.20.mlp.down_proj.weight": "model_3.safetensors", + "model.layers.20.experts.gate_up_proj": "model_3.safetensors", + "model.layers.20.experts.down_proj": "model_3.safetensors", + "model.layers.20.layer_scalar": "model_4.safetensors", + "model.layers.21.input_layernorm.weight": "model_4.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model_4.safetensors", + "model.layers.21.post_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.21.self_attn.q_norm.weight": "model_4.safetensors", + "model.layers.21.self_attn.k_norm.weight": "model_4.safetensors", + "model.layers.21.pre_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.21.post_feedforward_layernorm_1.weight": "model_4.safetensors", + "model.layers.21.router.scale": "model_4.safetensors", + "model.layers.21.router.per_expert_scale": "model_4.safetensors", + "model.layers.21.pre_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.21.post_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.21.router.proj.weight": "model_4.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model_4.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model_4.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model_4.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model_4.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model_4.safetensors", + "model.layers.21.mlp.up_proj.weight": "model_4.safetensors", + "model.layers.21.mlp.down_proj.weight": "model_4.safetensors", + "model.layers.21.experts.gate_up_proj": "model_4.safetensors", + "model.layers.21.experts.down_proj": "model_4.safetensors", + "model.layers.21.layer_scalar": "model_4.safetensors", + "model.layers.22.input_layernorm.weight": "model_4.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model_4.safetensors", + "model.layers.22.post_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.22.self_attn.q_norm.weight": "model_4.safetensors", + "model.layers.22.self_attn.k_norm.weight": "model_4.safetensors", + "model.layers.22.pre_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.22.post_feedforward_layernorm_1.weight": "model_4.safetensors", + "model.layers.22.router.scale": "model_4.safetensors", + "model.layers.22.router.per_expert_scale": "model_4.safetensors", + "model.layers.22.pre_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.22.post_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.22.router.proj.weight": "model_4.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model_4.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model_4.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model_4.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model_4.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model_4.safetensors", + "model.layers.22.mlp.up_proj.weight": "model_4.safetensors", + "model.layers.22.mlp.down_proj.weight": "model_4.safetensors", + "model.layers.22.experts.gate_up_proj": "model_4.safetensors", + "model.layers.22.experts.down_proj": "model_4.safetensors", + "model.layers.22.layer_scalar": "model_4.safetensors", + "model.layers.23.input_layernorm.weight": "model_4.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model_4.safetensors", + "model.layers.23.post_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.23.self_attn.q_norm.weight": "model_4.safetensors", + "model.layers.23.self_attn.k_norm.weight": "model_4.safetensors", + "model.layers.23.pre_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.23.post_feedforward_layernorm_1.weight": "model_4.safetensors", + "model.layers.23.router.scale": "model_4.safetensors", + "model.layers.23.router.per_expert_scale": "model_4.safetensors", + "model.layers.23.pre_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.23.post_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.23.router.proj.weight": "model_4.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model_4.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model_4.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model_4.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model_4.safetensors", + "model.layers.23.mlp.up_proj.weight": "model_4.safetensors", + "model.layers.23.mlp.down_proj.weight": "model_4.safetensors", + "model.layers.23.experts.gate_up_proj": "model_4.safetensors", + "model.layers.23.experts.down_proj": "model_4.safetensors", + "model.layers.23.layer_scalar": "model_4.safetensors", + "model.layers.24.input_layernorm.weight": "model_4.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model_4.safetensors", + "model.layers.24.post_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.24.self_attn.q_norm.weight": "model_4.safetensors", + "model.layers.24.self_attn.k_norm.weight": "model_4.safetensors", + "model.layers.24.pre_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.24.post_feedforward_layernorm_1.weight": "model_4.safetensors", + "model.layers.24.router.scale": "model_4.safetensors", + "model.layers.24.router.per_expert_scale": "model_4.safetensors", + "model.layers.24.pre_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.24.post_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.24.router.proj.weight": "model_4.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model_4.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model_4.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model_4.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model_4.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model_4.safetensors", + "model.layers.24.mlp.up_proj.weight": "model_4.safetensors", + "model.layers.24.mlp.down_proj.weight": "model_4.safetensors", + "model.layers.24.experts.gate_up_proj": "model_4.safetensors", + "model.layers.24.experts.down_proj": "model_4.safetensors", + "model.layers.24.layer_scalar": "model_4.safetensors", + "model.layers.25.input_layernorm.weight": "model_4.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model_4.safetensors", + "model.layers.25.post_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.25.self_attn.q_norm.weight": "model_4.safetensors", + "model.layers.25.self_attn.k_norm.weight": "model_4.safetensors", + "model.layers.25.pre_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.25.post_feedforward_layernorm_1.weight": "model_4.safetensors", + "model.layers.25.router.scale": "model_4.safetensors", + "model.layers.25.router.per_expert_scale": "model_4.safetensors", + "model.layers.25.pre_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.25.post_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.25.router.proj.weight": "model_4.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model_4.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model_4.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model_4.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model_4.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model_4.safetensors", + "model.layers.25.mlp.up_proj.weight": "model_4.safetensors", + "model.layers.25.mlp.down_proj.weight": "model_4.safetensors", + "model.layers.25.experts.gate_up_proj": "model_4.safetensors", + "model.layers.25.experts.down_proj": "model_4.safetensors", + "model.layers.25.layer_scalar": "model_4.safetensors", + "model.layers.26.input_layernorm.weight": "model_4.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model_4.safetensors", + "model.layers.26.post_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.26.self_attn.q_norm.weight": "model_4.safetensors", + "model.layers.26.self_attn.k_norm.weight": "model_4.safetensors", + "model.layers.26.pre_feedforward_layernorm.weight": "model_4.safetensors", + "model.layers.26.post_feedforward_layernorm_1.weight": "model_4.safetensors", + "model.layers.26.router.scale": "model_4.safetensors", + "model.layers.26.router.per_expert_scale": "model_4.safetensors", + "model.layers.26.pre_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.26.post_feedforward_layernorm_2.weight": "model_4.safetensors", + "model.layers.26.router.proj.weight": "model_4.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model_4.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model_4.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model_4.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model_4.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model_4.safetensors", + "model.layers.26.mlp.up_proj.weight": "model_4.safetensors", + "model.layers.26.mlp.down_proj.weight": "model_4.safetensors", + "model.layers.26.experts.gate_up_proj": "model_4.safetensors", + "model.layers.26.experts.down_proj": "model_5.safetensors", + "model.layers.26.layer_scalar": "model_5.safetensors", + "model.layers.27.input_layernorm.weight": "model_5.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model_5.safetensors", + "model.layers.27.post_feedforward_layernorm.weight": "model_5.safetensors", + "model.layers.27.self_attn.q_norm.weight": "model_5.safetensors", + "model.layers.27.self_attn.k_norm.weight": "model_5.safetensors", + "model.layers.27.pre_feedforward_layernorm.weight": "model_5.safetensors", + "model.layers.27.post_feedforward_layernorm_1.weight": "model_5.safetensors", + "model.layers.27.router.scale": "model_5.safetensors", + "model.layers.27.router.per_expert_scale": "model_5.safetensors", + "model.layers.27.pre_feedforward_layernorm_2.weight": "model_5.safetensors", + "model.layers.27.post_feedforward_layernorm_2.weight": "model_5.safetensors", + "model.layers.27.router.proj.weight": "model_5.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model_5.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model_5.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model_5.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model_5.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model_5.safetensors", + "model.layers.27.mlp.up_proj.weight": "model_5.safetensors", + "model.layers.27.mlp.down_proj.weight": "model_5.safetensors", + "model.layers.27.experts.gate_up_proj": "model_5.safetensors", + "model.layers.27.experts.down_proj": "model_5.safetensors", + "model.layers.27.layer_scalar": "model_5.safetensors", + "model.layers.28.input_layernorm.weight": "model_5.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model_5.safetensors", + "model.layers.28.post_feedforward_layernorm.weight": "model_5.safetensors", + "model.layers.28.self_attn.q_norm.weight": "model_5.safetensors", + "model.layers.28.self_attn.k_norm.weight": "model_5.safetensors", + "model.layers.28.pre_feedforward_layernorm.weight": "model_5.safetensors", + "model.layers.28.post_feedforward_layernorm_1.weight": "model_5.safetensors", + "model.layers.28.router.scale": "model_5.safetensors", + "model.layers.28.router.per_expert_scale": "model_5.safetensors", + "model.layers.28.pre_feedforward_layernorm_2.weight": "model_5.safetensors", + "model.layers.28.post_feedforward_layernorm_2.weight": "model_5.safetensors", + "model.layers.28.router.proj.weight": "model_5.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model_5.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model_5.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model_5.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model_5.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model_5.safetensors", + "model.layers.28.mlp.up_proj.weight": "model_5.safetensors", + "model.layers.28.mlp.down_proj.weight": "model_5.safetensors", + "model.layers.28.experts.gate_up_proj": "model_5.safetensors", + "model.layers.28.experts.down_proj": "model_5.safetensors", + "model.layers.28.layer_scalar": "model_5.safetensors", + "model.layers.29.input_layernorm.weight": "model_5.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model_5.safetensors", + "model.layers.29.post_feedforward_layernorm.weight": "model_5.safetensors", + "model.layers.29.self_attn.q_norm.weight": "model_5.safetensors", + "model.layers.29.self_attn.k_norm.weight": "model_5.safetensors", + "model.layers.29.pre_feedforward_layernorm.weight": "model_5.safetensors", + "model.layers.29.post_feedforward_layernorm_1.weight": "model_5.safetensors", + "model.layers.29.router.scale": "model_5.safetensors", + "model.layers.29.router.per_expert_scale": "model_5.safetensors", + "model.layers.29.pre_feedforward_layernorm_2.weight": "model_5.safetensors", + "model.layers.29.post_feedforward_layernorm_2.weight": "model_5.safetensors", + "model.layers.29.router.proj.weight": "model_5.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model_5.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model_5.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model_5.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model_5.safetensors", + "model.layers.29.mlp.up_proj.weight": "model_5.safetensors", + "model.layers.29.mlp.down_proj.weight": "model_5.safetensors", + "model.layers.29.experts.gate_up_proj": "model_5.safetensors", + "model.layers.29.experts.down_proj": "model_5.safetensors", + "model.layers.29.layer_scalar": "model_5.safetensors", + "model.norm.weight": "model_5.safetensors" + } +} \ No newline at end of file diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/70/model_0.safetensors b/26_09_08_sft_email_v4_v3/export/gemma4/70/model_0.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e4c8017a8f1281e721ceffc3b2d85756790554c7 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/70/model_0.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d193154bc227e9b768dae5398a411cc8fe1d490f8eba3df763ea15805c9f517 +size 18219689440 diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/70/model_1.safetensors b/26_09_08_sft_email_v4_v3/export/gemma4/70/model_1.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6f120da217becde8741f01d78617ffcbcad7eaaf --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/70/model_1.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e036e96a3b276aea68fea94af6dfe1b109dca14d20c43f877a25dac09cc00f5b +size 17354616388 diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/70/model_2.safetensors b/26_09_08_sft_email_v4_v3/export/gemma4/70/model_2.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..dbd2c37f9f5f7847716799ea2e2bab344a7a8a24 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/70/model_2.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b5ffba35f5720f0136ad3515f9eaac87883b0b498bbb037a0b01cab0e3816bd +size 18580955576 diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/70/model_3.safetensors b/26_09_08_sft_email_v4_v3/export/gemma4/70/model_3.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2045b2613670b7e06468715e9f32c180dc336beb --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/70/model_3.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81bbb0af18418e2e9606acc0f6366e2b8e58b5a2068b0315b872f22fd04d6920 +size 17354616500 diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/70/model_4.safetensors b/26_09_08_sft_email_v4_v3/export/gemma4/70/model_4.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cac3e0ec448e6f230969b5d479bb9bec7f09b9df --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/70/model_4.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6654be93b2273f19ce97c74ae00bddbbf25abb1d178edb8cd5354d3ebb277a69 +size 18580955552 diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/70/model_5.safetensors b/26_09_08_sft_email_v4_v3/export/gemma4/70/model_5.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0c6c7b4ecd595ba43c199f348a5e3239721e66c1 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/70/model_5.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a792bbb9303b8519c827e824db7aaa34e8932ee67b1b18c17832f74ec492942c +size 10841863000 diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/70/ok b/26_09_08_sft_email_v4_v3/export/gemma4/70/ok new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/70/tokenizer.json b/26_09_08_sft_email_v4_v3/export/gemma4/70/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..1ff9f3e3439a939b971f9919e821bf87e835a503 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/70/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc8d3a0ce36466ccc1278bf987df5f71db1719b9ca6b4118264f45cb627bfe0f +size 32169626 diff --git a/26_09_08_sft_email_v4_v3/export/gemma4/70/tokenizer_config.json b/26_09_08_sft_email_v4_v3/export/gemma4/70/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..375b25dc8be85705251e41be1c25310d24932051 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/export/gemma4/70/tokenizer_config.json @@ -0,0 +1,74 @@ +{ + "audio_token": "<|audio|>", + "backend": "tokenizers", + "boa_token": "<|audio>", + "boi_token": "<|image>", + "bos_token": "", + "eoa_token": "", + "eoc_token": "", + "eoi_token": "", + "eos_token": "", + "eot_token": "", + "escape_token": "<|\"|>", + "etc_token": "", + "etd_token": "", + "etr_token": "", + "extra_special_tokens": [ + "<|video|>" + ], + "image_token": "<|image|>", + "mask_token": "", + "model_max_length": 1000000000000000019884624838656, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma4Processor", + "response_schema": { + "type": "object", + "properties": { + "role": { + "const": "assistant" + }, + "thinking": { + "type": "string" + }, + "content": { + "type": "string" + }, + "tool_calls": { + "x-regex-iterator": "<\\|tool_call>(.*?)", + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "const": "function" + }, + "function": { + "type": "object", + "x-regex": "call\\:(?P\\w+)(?P\\{.*\\})", + "properties": { + "name": { + "type": "string" + }, + "arguments": { + "type": "object", + "x-parser": "gemma4-tool-call", + "additionalProperties": {} + } + } + } + } + } + } + }, + "x-regex": "(\\<\\|channel\\>thought\\n(?P.*?)\\)?(?P\\<\\|tool_call\\>.*\\)?(?P(?:(?!\\)(?!\\<\\|tool_response\\>).)+)?(?:\\|\\<\\|tool_response\\>)?" + }, + "soc_token": "<|channel>", + "sot_token": "<|turn>", + "stc_token": "<|tool_call>", + "std_token": "<|tool>", + "str_token": "<|tool_response>", + "think_token": "<|think|>", + "tokenizer_class": "GemmaTokenizer", + "unk_token": "" +} diff --git a/26_09_08_sft_email_v4_v3/model_config.yaml b/26_09_08_sft_email_v4_v3/model_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aa92bd2f7b55f9ab87b127564edfc7c753eecf70 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/model_config.yaml @@ -0,0 +1,87 @@ +type: train_gpt +run: + experiment_dir: /mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3 +model: + type: gpt + base_model: + embeddings: + vocab_parallel: false + decoder: + type: pattern + blocks: + full_attention: + mixer: + implementation: sdpa + mlp: + type: hybrid_moe + dense: + recompute_level: full + routed: + recompute_level: full + auxiliary_loss_coefficient: 0.0 + sliding_attention: + mixer: + implementation: flash + mlp: + type: hybrid_moe + dense: + recompute_level: full + routed: + recompute_level: full + auxiliary_loss_coefficient: 0.0 + head: + cross_entropy_splits: 8 + multi_stage: + zero_stage: 3 + distributed: + tensor_parallel: 2 + sequence_tensor_parallel: true + sequence_data_parallel: 4 + timeout: 3600.0 + compute_dtype: bfloat16 +pretrained: + format: gemma4 + model_weights: true + path: /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm + load_config: model +training: + evaluators: {} + logs: + interval: 1 + export: + format: gemma4 + interval: 70 + wandb: + project_name: gemma-sft + entity_name: corellm + train_iters: 210 + num_workers: 0 +schedule: + depth_first_micro_batches: 16 + breadth_first_micro_batches: 1 +data: + micro_batch_size: 32768 + maximum_document_length: 32768 + truncate_documents: false + datasets: + training: + type: blended + name: gemma_replay_and_automation_bench + datasets: + - type: file + path: /mnt/queue1/shashank/exp/data/26_09_04_sft_email_v3/fastllm/fast_llm_config.yaml + - type: file + path: /mnt/queue1/shashank/exp/data/26_09_08_sft_email_v4/fastllm/fast_llm_config.yaml + - type: file + path: /mnt/queue1/shruthan/data/gemma_sft_replay/10k/fastllm/fast_llm_config.yaml + weights: + - 19956395 + - 43688561 + - 69321955 +optimizer: + learning_rate: + base: 2.0e-05 + decay_style: cosine + warmup_iterations: 7 + decay_iterations: 210 + weight_decay: 0.01 diff --git a/26_09_08_sft_email_v4_v3/post_training.sh b/26_09_08_sft_email_v4_v3/post_training.sh new file mode 100644 index 0000000000000000000000000000000000000000..47a5c43b9b70c145924952daac2a98d85a776180 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/post_training.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +cp /mnt/queue1/shashank/exp/sft_gemma4_decoder_only_configs/* /mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3/export/gemma4/70/ +cp /mnt/queue1/shashank/exp/sft_gemma4_decoder_only_configs/* /mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3/export/gemma4/140/ +cp /mnt/queue1/shashank/exp/sft_gemma4_decoder_only_configs/* /mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3/export/gemma4/210/ diff --git a/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_00.txt b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_00.txt new file mode 100644 index 0000000000000000000000000000000000000000..cd2fa5d77b05deb675c6516c6c0f090ac2a1fe1c --- /dev/null +++ b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_00.txt @@ -0,0 +1,590 @@ +2026-09-08 11:16:27,114 [Rank 00] Creating model... +2026-09-08 11:16:28,703 [Rank 00] Splitting the model into 32 stages... +2026-09-08 11:16:28,719 [Rank 00] Total parameters: 12,991,426,590 +2026-09-08 11:16:28,719 [Rank 00] Weight buffer placement: +{1: 0, 3: 0, 5: 0, 7: 0, 9: 0, 11: 0, 13: 0, 15: 0, 17: 0, 19: 0, 21: 0, 23: 0, 25: 0, 27: 0, 29: 0, 31: 0, 2: 1, 4: 1, 6: 1, 8: 1, 10: 1, 12: 1, 14: 1, 16: 1, 18: 1, 20: 1, 22: 1, 24: 1, 26: 1, 28: 1, 30: 1, 0: 2} +2026-09-08 11:16:28,720 [Rank 00] Grad buffer placement: +{1: 0, 3: 0, 5: 0, 7: 0, 9: 0, 11: 0, 13: 0, 15: 0, 17: 0, 19: 0, 21: 0, 23: 0, 25: 0, 27: 0, 29: 0, 31: 0, 2: 1, 4: 1, 6: 1, 8: 1, 10: 1, 12: 1, 14: 1, 16: 1, 18: 1, 20: 1, 22: 1, 24: 1, 26: 1, 28: 1, 30: 1, 0: 2} +2026-09-08 11:16:28,720 [Rank 00] Command run: +/app/fast_llm/cli.py train gpt --config /mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3/model_config.yaml +2026-09-08 11:16:28,731 [Rank 00] +------- fast_llm.models.gpt.config.GPTTrainerConfig -------- +type: train_gpt +run: + experiment_dir: /mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3 +model: + type: gpt + base_model: + decoder: + type: pattern + blocks: + full_attention: + type: decoder + mixer: + type: attention + rotary: + type: proportional + theta: 1000000.0 + partial_rotary_factor: 0.25 + heads: 16 + head_groups: 2 + head_size: 512 + add_linear_biases: false + causal: true + dropout: 0.0 + softmax_scale_power: 0.0 + implementation: sdpa + query_norm: + type: rms_norm + epsilon: 1.0e-06 + zero_centered: false + key_norm: + type: rms_norm + epsilon: 1.0e-06 + zero_centered: false + value_norm: + type: fixed_rms_norm + epsilon: 1.0e-06 + shared_key_value: true + mlp: + type: hybrid_moe + pre_norm: null + post_norm: null + dense: + type: mlp + pre_norm: + type: rms_norm + epsilon: 1.0e-06 + zero_centered: false + post_norm: + type: rms_norm + epsilon: 1.0e-06 + zero_centered: false + intermediate_size: 2112 + add_linear_biases: false + gated: true + activation: gelu + recompute_level: full + routed: + type: moe + pre_norm: + type: rms_norm + epsilon: 1.0e-06 + zero_centered: false + post_norm: + type: rms_norm + epsilon: 1.0e-06 + zero_centered: false + intermediate_size: 704 + add_linear_biases: false + gated: true + activation: gelu + recompute_level: full + router_normalization: + type: fixed_rms_norm + epsilon: 1.0e-06 + router_scale: + enabled: true + router_input_scale: 0.018844459036110227 + router_per_expert_scale: + enabled: true + experts: 128 + shared_experts: 0 + experts_per_token: 8 + routing: aux_loss + auxiliary_loss_coefficient: 0.0 + normalization: + type: rms_norm + epsilon: 1.0e-06 + zero_centered: false + pre_mixer_normalization: null + pre_mlp_normalization: + type: none + post_mixer_normalization: + type: rms_norm + epsilon: 1.0e-06 + zero_centered: false + post_mlp_normalization: + type: rms_norm + epsilon: 1.0e-06 + zero_centered: false + output_scale: + lr_scale: 0.0 + enabled: true + sliding_attention: + type: decoder + mixer: + type: attention + rotary: + type: default + theta: 10000.0 + heads: 16 + head_groups: 8 + head_size: 256 + add_linear_biases: false + causal: true + dropout: 0.0 + window_size: 1024 + softmax_scale_power: 0.0 + implementation: flash + query_norm: + type: rms_norm + epsilon: 1.0e-06 + zero_centered: false + key_norm: + type: rms_norm + epsilon: 1.0e-06 + zero_centered: false + value_norm: + type: fixed_rms_norm + epsilon: 1.0e-06 + shared_key_value: false + mlp: + type: hybrid_moe + pre_norm: null + post_norm: null + dense: + type: mlp + pre_norm: + type: rms_norm + epsilon: 1.0e-06 + zero_centered: false + post_norm: + type: rms_norm + epsilon: 1.0e-06 + zero_centered: false + intermediate_size: 2112 + add_linear_biases: false + gated: true + activation: gelu + recompute_level: full + routed: + type: moe + pre_norm: + type: rms_norm + epsilon: 1.0e-06 + zero_centered: false + post_norm: + type: rms_norm + epsilon: 1.0e-06 + zero_centered: false + intermediate_size: 704 + add_linear_biases: false + gated: true + activation: gelu + recompute_level: full + router_normalization: + type: fixed_rms_norm + epsilon: 1.0e-06 + router_scale: + enabled: true + router_input_scale: 0.018844459036110227 + router_per_expert_scale: + enabled: true + experts: 128 + shared_experts: 0 + experts_per_token: 8 + routing: aux_loss + auxiliary_loss_coefficient: 0.0 + normalization: + type: rms_norm + epsilon: 1.0e-06 + zero_centered: false + pre_mixer_normalization: null + pre_mlp_normalization: + type: none + post_mixer_normalization: + type: rms_norm + epsilon: 1.0e-06 + zero_centered: false + post_mlp_normalization: + type: rms_norm + epsilon: 1.0e-06 + zero_centered: false + output_scale: + lr_scale: 0.0 + enabled: true + pattern: + - sliding_attention + - sliding_attention + - sliding_attention + - sliding_attention + - sliding_attention + - full_attention + - sliding_attention + - sliding_attention + - sliding_attention + - sliding_attention + - sliding_attention + - full_attention + - sliding_attention + - sliding_attention + - sliding_attention + - sliding_attention + - sliding_attention + - full_attention + - sliding_attention + - sliding_attention + - sliding_attention + - sliding_attention + - sliding_attention + - full_attention + - sliding_attention + - sliding_attention + - sliding_attention + - sliding_attention + - sliding_attention + - full_attention + num_blocks: 30 + embeddings: + vocab_size: 262144 + num_position_embeddings: 2048 + vocab_parallel: false + embedding_scale: 53.0659966456864 + head: + normalization: + type: rms_norm + epsilon: 1.0e-06 + zero_centered: false + losses: {} + cross_entropy_splits: 8 + final_logit_softcap: 30.0 + prediction_heads: 1 + tied_embedding_weight: true + hidden_size: 2816 + peft: + type: none + multi_stage: + zero_stage: 3 + distributed: + tensor_parallel: 2 + sequence_tensor_parallel: true + sequence_data_parallel: 4 + world_size: 32 + rank: 0 + local_world_size: 8 + timeout: 3600.0 + compute_dtype: bfloat16 +pretrained: + format: gemma4 + model_weights: true + path: /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm + load_config: model +training: + evaluators: {} + logs: + interval: 1 + export: + format: gemma4 + interval: 70 + wandb: + project_name: gemma-sft + entity_name: corellm + train_iters: 210 + num_workers: 0 +schedule: + depth_first_micro_batches: 16 + breadth_first_micro_batches: 1 +data: + micro_batch_size: 32768 + maximum_document_length: 32768 + truncate_documents: false + datasets: + training: + type: blended + name: gemma_replay_and_automation_bench + datasets: + - type: file + path: /mnt/queue1/shashank/exp/data/26_09_04_sft_email_v3/fastllm/fast_llm_config.yaml + - type: file + path: /mnt/queue1/shashank/exp/data/26_09_08_sft_email_v4/fastllm/fast_llm_config.yaml + - type: file + path: /mnt/queue1/shruthan/data/gemma_sft_replay/10k/fastllm/fast_llm_config.yaml + weights: + - 0.15008542238000852 + - 0.3285671651047079 + - 0.5213474125152836 +profiling: + cuda: false +optimizer: + learning_rate: + base: 2.0e-05 + decay_style: cosine + decay_iterations: 210 + warmup_iterations: 7 + weight_decay: 0.01 +--------------------------- end ---------------------------- +2026-09-08 11:16:31,678 [Rank 00] Setting up model... +2026-09-08 11:16:31,680 [Rank 00] >>> Allocating 3 weight buffers (2,975.25 MiB) +2026-09-08 11:16:31,687 [Rank 00] >>> Allocating 3 grad buffers (5,950.49 MiB) +2026-09-08 11:16:31,689 [Rank 00] >>> Allocating 4 shards (12,389.60 MiB) +2026-09-08 11:16:31,691 [Rank 00] Total allocated: 21,315.35 MiB +2026-09-08 11:16:31,737 [Rank 00] Preparing datasets... +2026-09-08 11:16:31,765 [Rank 00] Sampling dataset training. This may take several minutes. +2026-09-08 11:16:31,807 [Rank 00] > 26/3947 documents are longer than 32768 tokens and will be ignored. +2026-09-08 11:16:32,272 [Rank 00] Initializing training state from pretrained checkpoint at /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm (resetting optimizer state)... +2026-09-08 11:16:32,843 [Rank 00] Loading index from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model.safetensors.index.json +2026-09-08 11:16:32,848 [Rank 00] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00001-of-00002.safetensors +2026-09-08 11:25:08,562 [Rank 00] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00002-of-00002.safetensors +2026-09-08 11:25:26,458 [Rank 00] 811,965,856 state entries loaded successfully +2026-09-08 11:25:26,549 [Rank 00] done with setup ... +2026-09-08 11:25:26,550 [Rank 00] After initial setup: allocated 21,316.49 MiB | max allocated 22,768.49 MiB | reserved 23,696.00 MiB | max reserved 23,696.00 MiB | global max reserved 0.00 MiB +2026-09-08 11:25:26,550 [Rank 00] Initializing training dataset iterator from sample 0... +2026-09-08 11:25:26,551 [Rank 00] Training ... +2026-09-08 11:25:29,455 [Rank 00] Data loading took 1,069.76 ms +2026-09-08 11:26:42,945 [Rank 00] training @ step 1/ 210 | consumed tokens: 2,097,152 | batch size: 2097152 | step time: 76215.60 ms | throughput: 22027.92 tflop/s (model) | 27328.75 tflop/s (hardware) | 859.88 tokens/s/gpu | Memory allocated 21,478.56 MiB | max allocated 54,787.63 MiB | reserved 61,406.00 MiB | max reserved 61,406.00 MiB | global max reserved 23,696.00 MiB | learning rate: 2.857e-06 | loss scale: 1 | grad norm: 9.0255 | skipped iterations: 0 | nan iterations: 0 | average step time 76215.60 ms | remaining 4:25:29 | completion 2026-09-08 15:52:12 (0.48 %) | load_balancing_loss: 844.21033 | load_balancing_loss: nan | lm_head_loss: 0.34335 | run: 0 +2026-09-08 11:27:09,829 [Rank 00] training @ step 2/ 210 | consumed tokens: 4,194,304 | batch size: 2097152 | step time: 27042.08 ms | throughput: 62083.65 tflop/s (model) | 77023.56 tflop/s (hardware) | 2423.48 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,788.35 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,406.00 MiB | learning rate: 5.714e-06 | loss scale: 1 | grad norm: 22.4850 | skipped iterations: 0 | nan iterations: 0 | average step time 51628.84 ms | remaining 2:58:59 | completion 2026-09-08 14:26:09 (0.95 %) | load_balancing_loss: 843.64227 | load_balancing_loss: nan | lm_head_loss: 0.31742 | run: 0 +2026-09-08 11:27:36,271 [Rank 00] training @ step 3/ 210 | consumed tokens: 6,291,456 | batch size: 2097152 | step time: 26442.21 ms | throughput: 63492.08 tflop/s (model) | 78770.92 tflop/s (hardware) | 2478.46 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.42 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 8.571e-06 | loss scale: 1 | grad norm: 5.7799 | skipped iterations: 0 | nan iterations: 0 | average step time 43233.29 ms | remaining 2:29:09 | completion 2026-09-08 13:56:46 (1.43 %) | load_balancing_loss: 836.33130 | load_balancing_loss: nan | lm_head_loss: 0.29141 | run: 0 +2026-09-08 11:28:04,817 [Rank 00] training @ step 4/ 210 | consumed tokens: 8,388,608 | batch size: 2097152 | step time: 28545.90 ms | throughput: 58813.03 tflop/s (model) | 72965.90 tflop/s (hardware) | 2295.81 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,788.19 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.143e-05 | loss scale: 1 | grad norm: 4.5661 | skipped iterations: 0 | nan iterations: 0 | average step time 39561.44 ms | remaining 2:15:50 | completion 2026-09-08 13:43:54 (1.90 %) | load_balancing_loss: 834.74432 | load_balancing_loss: nan | lm_head_loss: 0.25728 | run: 0 +2026-09-08 11:28:33,517 [Rank 00] training @ step 5/ 210 | consumed tokens: 10,485,760 | batch size: 2097152 | step time: 28700.30 ms | throughput: 58496.63 tflop/s (model) | 72573.36 tflop/s (hardware) | 2283.46 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.81 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.429e-05 | loss scale: 1 | grad norm: 1.9863 | skipped iterations: 0 | nan iterations: 0 | average step time 37389.21 ms | remaining 2:07:45 | completion 2026-09-08 13:36:18 (2.38 %) | load_balancing_loss: 857.50330 | load_balancing_loss: nan | lm_head_loss: 0.23458 | run: 0 +2026-09-08 11:29:00,394 [Rank 00] training @ step 6/ 210 | consumed tokens: 12,582,912 | batch size: 2097152 | step time: 26876.93 ms | throughput: 62465.13 tflop/s (model) | 77496.84 tflop/s (hardware) | 2438.37 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.80 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.714e-05 | loss scale: 1 | grad norm: 5.0986 | skipped iterations: 0 | nan iterations: 0 | average step time 35637.17 ms | remaining 2:01:10 | completion 2026-09-08 13:30:10 (2.86 %) | load_balancing_loss: 838.99622 | load_balancing_loss: nan | lm_head_loss: 0.24859 | run: 0 +2026-09-08 11:29:27,437 [Rank 00] training @ step 7/ 210 | consumed tokens: 14,680,064 | batch size: 2097152 | step time: 27043.69 ms | throughput: 62079.93 tflop/s (model) | 77018.95 tflop/s (hardware) | 2423.34 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.00 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 2.000e-05 | loss scale: 1 | grad norm: 3.9877 | skipped iterations: 0 | nan iterations: 0 | average step time 34409.53 ms | remaining 1:56:25 | completion 2026-09-08 13:25:53 (3.33 %) | load_balancing_loss: 844.65076 | load_balancing_loss: nan | lm_head_loss: 0.21820 | run: 0 +2026-09-08 11:29:54,821 [Rank 00] training @ step 8/ 210 | consumed tokens: 16,777,216 | batch size: 2097152 | step time: 27383.88 ms | throughput: 61308.73 tflop/s (model) | 76062.16 tflop/s (hardware) | 2393.23 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.48 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 2.000e-05 | loss scale: 1 | grad norm: 5.7364 | skipped iterations: 0 | nan iterations: 0 | average step time 33531.32 ms | remaining 1:52:53 | completion 2026-09-08 13:22:48 (3.81 %) | load_balancing_loss: 827.87720 | load_balancing_loss: nan | lm_head_loss: 0.23878 | run: 0 +2026-09-08 11:30:22,379 [Rank 00] training @ step 9/ 210 | consumed tokens: 18,874,368 | batch size: 2097152 | step time: 27557.64 ms | throughput: 60922.15 tflop/s (model) | 75582.56 tflop/s (hardware) | 2378.14 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.12 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 2.000e-05 | loss scale: 1 | grad norm: 64.1650 | skipped iterations: 0 | nan iterations: 0 | average step time 32867.58 ms | remaining 1:50:06 | completion 2026-09-08 13:20:29 (4.29 %) | load_balancing_loss: 815.95013 | load_balancing_loss: nan | lm_head_loss: 0.24833 | run: 0 +2026-09-08 11:30:48,800 [Rank 00] training @ step 10/ 210 | consumed tokens: 20,971,520 | batch size: 2097152 | step time: 26421.14 ms | throughput: 63542.71 tflop/s (model) | 78833.74 tflop/s (hardware) | 2480.44 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.68 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.999e-05 | loss scale: 1 | grad norm: 2.4194 | skipped iterations: 0 | nan iterations: 0 | average step time 32222.93 ms | remaining 1:47:25 | completion 2026-09-08 13:18:13 (4.76 %) | load_balancing_loss: 831.75330 | load_balancing_loss: nan | lm_head_loss: 0.23743 | run: 0 +2026-09-08 11:31:19,173 [Rank 00] training @ step 11/ 210 | consumed tokens: 23,068,672 | batch size: 2097152 | step time: 30373.25 ms | throughput: 55274.65 tflop/s (model) | 68576.04 tflop/s (hardware) | 2157.69 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.39 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.998e-05 | loss scale: 1 | grad norm: 9.2401 | skipped iterations: 0 | nan iterations: 0 | average step time 32054.78 ms | remaining 1:46:19 | completion 2026-09-08 13:17:38 (5.24 %) | load_balancing_loss: 845.51526 | load_balancing_loss: nan | lm_head_loss: 0.21192 | run: 0 +2026-09-08 11:31:47,293 [Rank 00] training @ step 12/ 210 | consumed tokens: 25,165,824 | batch size: 2097152 | step time: 28119.28 ms | throughput: 59705.32 tflop/s (model) | 74072.91 tflop/s (hardware) | 2330.64 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.63 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.997e-05 | loss scale: 1 | grad norm: 6.6991 | skipped iterations: 0 | nan iterations: 0 | average step time 31726.82 ms | remaining 1:44:42 | completion 2026-09-08 13:16:29 (5.71 %) | load_balancing_loss: 825.45508 | load_balancing_loss: nan | lm_head_loss: 0.20460 | run: 0 +2026-09-08 11:32:16,701 [Rank 00] training @ step 13/ 210 | consumed tokens: 27,262,976 | batch size: 2097152 | step time: 29408.42 ms | throughput: 57088.09 tflop/s (model) | 70825.86 tflop/s (hardware) | 2228.48 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,788.02 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.996e-05 | loss scale: 1 | grad norm: 29.9379 | skipped iterations: 0 | nan iterations: 0 | average step time 31548.48 ms | remaining 1:43:35 | completion 2026-09-08 13:15:52 (6.19 %) | load_balancing_loss: 827.40405 | load_balancing_loss: nan | lm_head_loss: 0.22295 | run: 0 +2026-09-08 11:32:43,702 [Rank 00] training @ step 14/ 210 | consumed tokens: 29,360,128 | batch size: 2097152 | step time: 27001.17 ms | throughput: 62177.69 tflop/s (model) | 77140.24 tflop/s (hardware) | 2427.15 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.65 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.994e-05 | loss scale: 1 | grad norm: 107.5821 | skipped iterations: 0 | nan iterations: 0 | average step time 31223.68 ms | remaining 1:42:00 | completion 2026-09-08 13:14:44 (6.67 %) | load_balancing_loss: 828.73895 | load_balancing_loss: nan | lm_head_loss: 0.21661 | run: 0 +2026-09-08 11:33:09,728 [Rank 00] training @ step 15/ 210 | consumed tokens: 31,457,280 | batch size: 2097152 | step time: 26025.32 ms | throughput: 64509.12 tflop/s (model) | 80032.71 tflop/s (hardware) | 2518.16 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.65 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.992e-05 | loss scale: 1 | grad norm: 3.3808 | skipped iterations: 0 | nan iterations: 0 | average step time 30877.12 ms | remaining 1:40:21 | completion 2026-09-08 13:13:31 (7.14 %) | load_balancing_loss: 823.92493 | load_balancing_loss: nan | lm_head_loss: 0.22511 | run: 0 +2026-09-08 11:33:37,242 [Rank 00] training @ step 16/ 210 | consumed tokens: 33,554,432 | batch size: 2097152 | step time: 27514.50 ms | throughput: 61017.67 tflop/s (model) | 75701.06 tflop/s (hardware) | 2381.87 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.35 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.990e-05 | loss scale: 1 | grad norm: 7.1353 | skipped iterations: 0 | nan iterations: 0 | average step time 30666.96 ms | remaining 1:39:09 | completion 2026-09-08 13:12:47 (7.62 %) | load_balancing_loss: 827.53235 | load_balancing_loss: nan | lm_head_loss: 0.23604 | run: 0 +2026-09-08 11:34:03,548 [Rank 00] training @ step 17/ 210 | consumed tokens: 35,651,584 | batch size: 2097152 | step time: 26305.99 ms | throughput: 63820.86 tflop/s (model) | 79178.81 tflop/s (hardware) | 2491.30 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.22 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.988e-05 | loss scale: 1 | grad norm: 2.7748 | skipped iterations: 0 | nan iterations: 0 | average step time 30410.43 ms | remaining 1:37:49 | completion 2026-09-08 13:11:53 (8.10 %) | load_balancing_loss: 820.13684 | load_balancing_loss: nan | lm_head_loss: 0.21292 | run: 0 +2026-09-08 11:34:30,587 [Rank 00] training @ step 18/ 210 | consumed tokens: 37,748,736 | batch size: 2097152 | step time: 27038.49 ms | throughput: 62091.87 tflop/s (model) | 77033.76 tflop/s (hardware) | 2423.80 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.02 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.986e-05 | loss scale: 1 | grad norm: 1.2647 | skipped iterations: 0 | nan iterations: 0 | average step time 30223.10 ms | remaining 1:36:43 | completion 2026-09-08 13:11:13 (8.57 %) | load_balancing_loss: 824.92383 | load_balancing_loss: nan | lm_head_loss: 0.20826 | run: 0 +2026-09-08 11:34:57,947 [Rank 00] training @ step 19/ 210 | consumed tokens: 39,845,888 | batch size: 2097152 | step time: 27360.91 ms | throughput: 61360.19 tflop/s (model) | 76126.01 tflop/s (hardware) | 2395.24 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.87 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.983e-05 | loss scale: 1 | grad norm: 1.4047 | skipped iterations: 0 | nan iterations: 0 | average step time 30072.46 ms | remaining 1:35:44 | completion 2026-09-08 13:10:42 (9.05 %) | load_balancing_loss: 826.70776 | load_balancing_loss: nan | lm_head_loss: 0.19092 | run: 0 +2026-09-08 11:35:26,704 [Rank 00] training @ step 20/ 210 | consumed tokens: 41,943,040 | batch size: 2097152 | step time: 28756.72 ms | throughput: 58381.85 tflop/s (model) | 72430.95 tflop/s (hardware) | 2278.98 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.88 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.980e-05 | loss scale: 1 | grad norm: 6.3498 | skipped iterations: 0 | nan iterations: 0 | average step time 30006.67 ms | remaining 1:35:01 | completion 2026-09-08 13:10:28 (9.52 %) | load_balancing_loss: 823.23145 | load_balancing_loss: nan | lm_head_loss: 0.18898 | run: 0 +2026-09-08 11:35:53,720 [Rank 00] training @ step 21/ 210 | consumed tokens: 44,040,192 | batch size: 2097152 | step time: 27015.99 ms | throughput: 62143.60 tflop/s (model) | 77097.93 tflop/s (hardware) | 2425.82 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.61 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.977e-05 | loss scale: 1 | grad norm: 4.0043 | skipped iterations: 0 | nan iterations: 0 | average step time 29864.26 ms | remaining 1:34:04 | completion 2026-09-08 13:09:58 (10.00 %) | load_balancing_loss: 826.26306 | load_balancing_loss: nan | lm_head_loss: 0.18659 | run: 0 +2026-09-08 11:36:20,935 [Rank 00] training @ step 22/ 210 | consumed tokens: 46,137,344 | batch size: 2097152 | step time: 27214.57 ms | throughput: 61690.14 tflop/s (model) | 76535.35 tflop/s (hardware) | 2408.12 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.58 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.973e-05 | loss scale: 1 | grad norm: 4.7882 | skipped iterations: 0 | nan iterations: 0 | average step time 29743.82 ms | remaining 1:33:12 | completion 2026-09-08 13:09:33 (10.48 %) | load_balancing_loss: 821.25793 | load_balancing_loss: nan | lm_head_loss: 0.19665 | run: 0 +2026-09-08 11:36:49,298 [Rank 00] training @ step 23/ 210 | consumed tokens: 48,234,496 | batch size: 2097152 | step time: 28363.71 ms | throughput: 59190.80 tflop/s (model) | 73434.57 tflop/s (hardware) | 2310.56 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,788.02 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.970e-05 | loss scale: 1 | grad norm: 7.7885 | skipped iterations: 0 | nan iterations: 0 | average step time 29683.81 ms | remaining 1:32:31 | completion 2026-09-08 13:09:20 (10.95 %) | load_balancing_loss: 835.11407 | load_balancing_loss: nan | lm_head_loss: 0.19392 | run: 0 +2026-09-08 11:37:17,054 [Rank 00] training @ step 24/ 210 | consumed tokens: 50,331,648 | batch size: 2097152 | step time: 27755.88 ms | throughput: 60487.03 tflop/s (model) | 75042.73 tflop/s (hardware) | 2361.16 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.29 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.966e-05 | loss scale: 1 | grad norm: 1.5741 | skipped iterations: 0 | nan iterations: 0 | average step time 29603.48 ms | remaining 1:31:46 | completion 2026-09-08 13:09:03 (11.43 %) | load_balancing_loss: 816.77930 | load_balancing_loss: nan | lm_head_loss: 0.19971 | run: 0 +2026-09-08 11:37:45,069 [Rank 00] training @ step 25/ 210 | consumed tokens: 52,428,800 | batch size: 2097152 | step time: 28014.63 ms | throughput: 59928.35 tflop/s (model) | 74349.61 tflop/s (hardware) | 2339.35 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.14 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.961e-05 | loss scale: 1 | grad norm: 3.6965 | skipped iterations: 0 | nan iterations: 0 | average step time 29539.93 ms | remaining 1:31:05 | completion 2026-09-08 13:08:50 (11.90 %) | load_balancing_loss: 821.58691 | load_balancing_loss: nan | lm_head_loss: 0.20379 | run: 0 +2026-09-08 11:38:11,120 [Rank 00] training @ step 26/ 210 | consumed tokens: 54,525,952 | batch size: 2097152 | step time: 26050.59 ms | throughput: 64446.54 tflop/s (model) | 79955.07 tflop/s (hardware) | 2515.72 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.84 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.957e-05 | loss scale: 1 | grad norm: 2.8615 | skipped iterations: 0 | nan iterations: 0 | average step time 29405.72 ms | remaining 1:30:11 | completion 2026-09-08 13:08:22 (12.38 %) | load_balancing_loss: 825.46918 | load_balancing_loss: nan | lm_head_loss: 0.17676 | run: 0 +2026-09-08 11:38:39,525 [Rank 00] training @ step 27/ 210 | consumed tokens: 56,623,104 | batch size: 2097152 | step time: 28405.22 ms | throughput: 59104.30 tflop/s (model) | 73327.26 tflop/s (hardware) | 2307.18 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.57 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.952e-05 | loss scale: 1 | grad norm: 1.6831 | skipped iterations: 0 | nan iterations: 0 | average step time 29368.67 ms | remaining 1:29:34 | completion 2026-09-08 13:08:14 (12.86 %) | load_balancing_loss: 824.41699 | load_balancing_loss: nan | lm_head_loss: 0.19893 | run: 0 +2026-09-08 11:39:06,748 [Rank 00] training @ step 28/ 210 | consumed tokens: 58,720,256 | batch size: 2097152 | step time: 27223.26 ms | throughput: 61670.45 tflop/s (model) | 76510.93 tflop/s (hardware) | 2407.35 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.15 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.948e-05 | loss scale: 1 | grad norm: 7.4563 | skipped iterations: 0 | nan iterations: 0 | average step time 29292.05 ms | remaining 1:28:51 | completion 2026-09-08 13:07:58 (13.33 %) | load_balancing_loss: 817.97559 | load_balancing_loss: nan | lm_head_loss: 0.18216 | run: 0 +2026-09-08 11:39:34,163 [Rank 00] training @ step 29/ 210 | consumed tokens: 60,817,408 | batch size: 2097152 | step time: 27414.63 ms | throughput: 61239.97 tflop/s (model) | 75976.85 tflop/s (hardware) | 2390.55 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,788.09 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.943e-05 | loss scale: 1 | grad norm: 3.0791 | skipped iterations: 0 | nan iterations: 0 | average step time 29227.31 ms | remaining 1:28:10 | completion 2026-09-08 13:07:44 (13.81 %) | load_balancing_loss: 833.58362 | load_balancing_loss: nan | lm_head_loss: 0.18225 | run: 0 +2026-09-08 11:40:02,709 [Rank 00] training @ step 30/ 210 | consumed tokens: 62,914,560 | batch size: 2097152 | step time: 28546.11 ms | throughput: 58812.59 tflop/s (model) | 72965.35 tflop/s (hardware) | 2295.79 tokens/s/gpu | Memory allocated 21,478.56 MiB | max allocated 54,787.04 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.937e-05 | loss scale: 1 | grad norm: 2.3419 | skipped iterations: 0 | nan iterations: 0 | average step time 29204.60 ms | remaining 1:27:37 | completion 2026-09-08 13:07:40 (14.29 %) | load_balancing_loss: 823.84387 | load_balancing_loss: nan | lm_head_loss: 0.18617 | run: 0 +2026-09-08 11:40:31,977 [Rank 00] training @ step 31/ 210 | consumed tokens: 65,011,712 | batch size: 2097152 | step time: 29268.51 ms | throughput: 57361.00 tflop/s (model) | 71164.44 tflop/s (hardware) | 2239.13 tokens/s/gpu | Memory allocated 21,478.56 MiB | max allocated 54,787.30 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.932e-05 | loss scale: 1 | grad norm: 1.7602 | skipped iterations: 0 | nan iterations: 0 | average step time 29206.66 ms | remaining 1:27:08 | completion 2026-09-08 13:07:40 (14.76 %) | load_balancing_loss: 817.68359 | load_balancing_loss: nan | lm_head_loss: 0.18080 | run: 0 +2026-09-08 11:40:58,647 [Rank 00] training @ step 32/ 210 | consumed tokens: 67,108,864 | batch size: 2097152 | step time: 26669.79 ms | throughput: 62950.28 tflop/s (model) | 78098.73 tflop/s (hardware) | 2457.31 tokens/s/gpu | Memory allocated 21,478.56 MiB | max allocated 54,787.36 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.926e-05 | loss scale: 1 | grad norm: 3.9723 | skipped iterations: 0 | nan iterations: 0 | average step time 29127.38 ms | remaining 1:26:25 | completion 2026-09-08 13:07:23 (15.24 %) | load_balancing_loss: 810.33386 | load_balancing_loss: nan | lm_head_loss: 0.17901 | run: 0 +2026-09-08 11:41:26,453 [Rank 00] training @ step 33/ 210 | consumed tokens: 69,206,016 | batch size: 2097152 | step time: 27805.84 ms | throughput: 60378.34 tflop/s (model) | 74907.89 tflop/s (hardware) | 2356.91 tokens/s/gpu | Memory allocated 21,478.56 MiB | max allocated 54,788.01 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.920e-05 | loss scale: 1 | grad norm: 6.0343 | skipped iterations: 0 | nan iterations: 0 | average step time 29087.34 ms | remaining 1:25:48 | completion 2026-09-08 13:07:15 (15.71 %) | load_balancing_loss: 824.43219 | load_balancing_loss: nan | lm_head_loss: 0.19091 | run: 0 +2026-09-08 11:41:53,320 [Rank 00] training @ step 34/ 210 | consumed tokens: 71,303,168 | batch size: 2097152 | step time: 26867.29 ms | throughput: 62487.54 tflop/s (model) | 77524.65 tflop/s (hardware) | 2439.25 tokens/s/gpu | Memory allocated 21,478.56 MiB | max allocated 54,787.78 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.914e-05 | loss scale: 1 | grad norm: 4.5105 | skipped iterations: 0 | nan iterations: 0 | average step time 29022.04 ms | remaining 1:25:08 | completion 2026-09-08 13:07:01 (16.19 %) | load_balancing_loss: 824.51215 | load_balancing_loss: nan | lm_head_loss: 0.17671 | run: 0 +2026-09-08 11:42:21,478 [Rank 00] training @ step 35/ 210 | consumed tokens: 73,400,320 | batch size: 2097152 | step time: 28157.65 ms | throughput: 59623.97 tflop/s (model) | 73971.98 tflop/s (hardware) | 2327.47 tokens/s/gpu | Memory allocated 21,478.56 MiB | max allocated 54,788.16 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.908e-05 | loss scale: 1 | grad norm: 3.7438 | skipped iterations: 0 | nan iterations: 0 | average step time 28997.35 ms | remaining 1:24:35 | completion 2026-09-08 13:06:56 (16.67 %) | load_balancing_loss: 820.45386 | load_balancing_loss: nan | lm_head_loss: 0.17619 | run: 0 +2026-09-08 11:42:48,499 [Rank 00] training @ step 36/ 210 | consumed tokens: 75,497,472 | batch size: 2097152 | step time: 27021.06 ms | throughput: 62131.93 tflop/s (model) | 77083.46 tflop/s (hardware) | 2425.37 tokens/s/gpu | Memory allocated 21,478.56 MiB | max allocated 54,786.97 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.901e-05 | loss scale: 1 | grad norm: 2.9865 | skipped iterations: 0 | nan iterations: 0 | average step time 28942.45 ms | remaining 1:23:56 | completion 2026-09-08 13:06:44 (17.14 %) | load_balancing_loss: 816.39844 | load_balancing_loss: nan | lm_head_loss: 0.17077 | run: 0 +2026-09-08 11:43:16,043 [Rank 00] training @ step 37/ 210 | consumed tokens: 77,594,624 | batch size: 2097152 | step time: 27544.18 ms | throughput: 60951.93 tflop/s (model) | 75619.50 tflop/s (hardware) | 2379.31 tokens/s/gpu | Memory allocated 21,478.56 MiB | max allocated 54,787.63 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.894e-05 | loss scale: 1 | grad norm: 2.9934 | skipped iterations: 0 | nan iterations: 0 | average step time 28904.66 ms | remaining 1:23:21 | completion 2026-09-08 13:06:37 (17.62 %) | load_balancing_loss: 827.56372 | load_balancing_loss: nan | lm_head_loss: 0.17331 | run: 0 +2026-09-08 11:43:43,452 [Rank 00] training @ step 38/ 210 | consumed tokens: 79,691,776 | batch size: 2097152 | step time: 27408.82 ms | throughput: 61252.94 tflop/s (model) | 75992.94 tflop/s (hardware) | 2391.06 tokens/s/gpu | Memory allocated 21,478.56 MiB | max allocated 54,787.24 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.887e-05 | loss scale: 1 | grad norm: 3.6132 | skipped iterations: 0 | nan iterations: 0 | average step time 28865.29 ms | remaining 1:22:45 | completion 2026-09-08 13:06:28 (18.10 %) | load_balancing_loss: 812.28912 | load_balancing_loss: nan | lm_head_loss: 0.17314 | run: 0 +2026-09-08 11:44:10,240 [Rank 00] training @ step 39/ 210 | consumed tokens: 81,788,928 | batch size: 2097152 | step time: 26788.44 ms | throughput: 62671.46 tflop/s (model) | 77752.82 tflop/s (hardware) | 2446.43 tokens/s/gpu | Memory allocated 21,478.56 MiB | max allocated 54,787.87 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.880e-05 | loss scale: 1 | grad norm: 9.3502 | skipped iterations: 0 | nan iterations: 0 | average step time 28812.04 ms | remaining 1:22:07 | completion 2026-09-08 13:06:17 (18.57 %) | load_balancing_loss: 821.04541 | load_balancing_loss: nan | lm_head_loss: 0.16066 | run: 0 +2026-09-08 11:44:37,878 [Rank 00] training @ step 40/ 210 | consumed tokens: 83,886,080 | batch size: 2097152 | step time: 27637.72 ms | throughput: 60745.63 tflop/s (model) | 75363.56 tflop/s (hardware) | 2371.25 tokens/s/gpu | Memory allocated 21,478.56 MiB | max allocated 54,787.79 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.872e-05 | loss scale: 1 | grad norm: 3.3204 | skipped iterations: 0 | nan iterations: 0 | average step time 28782.68 ms | remaining 1:21:33 | completion 2026-09-08 13:06:11 (19.05 %) | load_balancing_loss: 823.14429 | load_balancing_loss: nan | lm_head_loss: 0.16647 | run: 0 +2026-09-08 11:45:03,808 [Rank 00] training @ step 41/ 210 | consumed tokens: 85,983,232 | batch size: 2097152 | step time: 25929.93 ms | throughput: 64746.44 tflop/s (model) | 80327.13 tflop/s (hardware) | 2527.43 tokens/s/gpu | Memory allocated 21,478.56 MiB | max allocated 54,787.65 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.865e-05 | loss scale: 1 | grad norm: 6.3801 | skipped iterations: 0 | nan iterations: 0 | average step time 28713.10 ms | remaining 1:20:53 | completion 2026-09-08 13:05:56 (19.52 %) | load_balancing_loss: 812.35095 | load_balancing_loss: nan | lm_head_loss: 0.18428 | run: 0 +2026-09-08 11:45:31,568 [Rank 00] training @ step 42/ 210 | consumed tokens: 88,080,384 | batch size: 2097152 | step time: 27760.21 ms | throughput: 60477.59 tflop/s (model) | 75031.01 tflop/s (hardware) | 2360.79 tokens/s/gpu | Memory allocated 21,478.56 MiB | max allocated 54,787.19 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.857e-05 | loss scale: 1 | grad norm: 14.6796 | skipped iterations: 0 | nan iterations: 0 | average step time 28690.42 ms | remaining 1:20:20 | completion 2026-09-08 13:05:52 (20.00 %) | load_balancing_loss: 809.08759 | load_balancing_loss: nan | lm_head_loss: 0.17825 | run: 0 +2026-09-08 11:45:58,874 [Rank 00] training @ step 43/ 210 | consumed tokens: 90,177,536 | batch size: 2097152 | step time: 27305.65 ms | throughput: 61484.37 tflop/s (model) | 76280.07 tflop/s (hardware) | 2400.09 tokens/s/gpu | Memory allocated 21,478.56 MiB | max allocated 54,788.01 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.849e-05 | loss scale: 1 | grad norm: 3.0971 | skipped iterations: 0 | nan iterations: 0 | average step time 28658.21 ms | remaining 1:19:46 | completion 2026-09-08 13:05:45 (20.48 %) | load_balancing_loss: 828.03882 | load_balancing_loss: nan | lm_head_loss: 0.16768 | run: 0 +2026-09-08 11:46:27,448 [Rank 00] training @ step 44/ 210 | consumed tokens: 92,274,688 | batch size: 2097152 | step time: 28573.78 ms | throughput: 58755.65 tflop/s (model) | 72894.70 tflop/s (hardware) | 2293.57 tokens/s/gpu | Memory allocated 21,478.56 MiB | max allocated 54,787.49 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.840e-05 | loss scale: 1 | grad norm: 6.0707 | skipped iterations: 0 | nan iterations: 0 | average step time 28656.29 ms | remaining 1:19:17 | completion 2026-09-08 13:05:44 (20.95 %) | load_balancing_loss: 800.21765 | load_balancing_loss: nan | lm_head_loss: 0.17126 | run: 0 +2026-09-08 11:46:54,058 [Rank 00] training @ step 45/ 210 | consumed tokens: 94,371,840 | batch size: 2097152 | step time: 26610.57 ms | throughput: 63090.37 tflop/s (model) | 78272.55 tflop/s (hardware) | 2462.78 tokens/s/gpu | Memory allocated 21,478.56 MiB | max allocated 54,787.15 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.832e-05 | loss scale: 1 | grad norm: 1.9195 | skipped iterations: 0 | nan iterations: 0 | average step time 28610.83 ms | remaining 1:18:41 | completion 2026-09-08 13:05:35 (21.43 %) | load_balancing_loss: 815.54523 | load_balancing_loss: nan | lm_head_loss: 0.17277 | run: 0 +2026-09-08 11:47:21,811 [Rank 00] training @ step 46/ 210 | consumed tokens: 96,468,992 | batch size: 2097152 | step time: 27752.66 ms | throughput: 60494.05 tflop/s (model) | 75051.43 tflop/s (hardware) | 2361.43 tokens/s/gpu | Memory allocated 21,478.56 MiB | max allocated 54,787.90 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.823e-05 | loss scale: 1 | grad norm: 3.1466 | skipped iterations: 0 | nan iterations: 0 | average step time 28592.18 ms | remaining 1:18:09 | completion 2026-09-08 13:05:31 (21.90 %) | load_balancing_loss: 801.84607 | load_balancing_loss: nan | lm_head_loss: 0.17242 | run: 0 +2026-09-08 11:47:50,390 [Rank 00] training @ step 47/ 210 | consumed tokens: 98,566,144 | batch size: 2097152 | step time: 28578.83 ms | throughput: 58745.26 tflop/s (model) | 72881.81 tflop/s (hardware) | 2293.17 tokens/s/gpu | Memory allocated 21,478.56 MiB | max allocated 54,787.56 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.814e-05 | loss scale: 1 | grad norm: 3.3170 | skipped iterations: 0 | nan iterations: 0 | average step time 28591.89 ms | remaining 1:17:40 | completion 2026-09-08 13:05:31 (22.38 %) | load_balancing_loss: 832.03973 | load_balancing_loss: nan | lm_head_loss: 0.16196 | run: 0 +2026-09-08 11:48:17,345 [Rank 00] training @ step 48/ 210 | consumed tokens: 100,663,296 | batch size: 2097152 | step time: 26955.21 ms | throughput: 62283.72 tflop/s (model) | 77271.77 tflop/s (hardware) | 2431.29 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.29 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.805e-05 | loss scale: 1 | grad norm: 4.4367 | skipped iterations: 0 | nan iterations: 0 | average step time 28557.79 ms | remaining 1:17:06 | completion 2026-09-08 13:05:24 (22.86 %) | load_balancing_loss: 823.22070 | load_balancing_loss: nan | lm_head_loss: 0.16719 | run: 0 +2026-09-08 11:48:44,228 [Rank 00] training @ step 49/ 210 | consumed tokens: 102,760,448 | batch size: 2097152 | step time: 26883.18 ms | throughput: 62450.60 tflop/s (model) | 77478.82 tflop/s (hardware) | 2437.81 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,788.34 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.796e-05 | loss scale: 1 | grad norm: 5.5229 | skipped iterations: 0 | nan iterations: 0 | average step time 28523.62 ms | remaining 1:16:32 | completion 2026-09-08 13:05:17 (23.33 %) | load_balancing_loss: 820.17090 | load_balancing_loss: nan | lm_head_loss: 0.15969 | run: 0 +2026-09-08 11:49:10,315 [Rank 00] training @ step 50/ 210 | consumed tokens: 104,857,600 | batch size: 2097152 | step time: 26086.69 ms | throughput: 64357.36 tflop/s (model) | 79844.42 tflop/s (hardware) | 2512.24 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.27 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.787e-05 | loss scale: 1 | grad norm: 9.9216 | skipped iterations: 0 | nan iterations: 0 | average step time 28474.88 ms | remaining 1:15:56 | completion 2026-09-08 13:05:06 (23.81 %) | load_balancing_loss: 830.27808 | load_balancing_loss: nan | lm_head_loss: 0.16237 | run: 0 +2026-09-08 11:49:35,853 [Rank 00] training @ step 51/ 210 | consumed tokens: 106,954,752 | batch size: 2097152 | step time: 25537.91 ms | throughput: 65740.34 tflop/s (model) | 81560.21 tflop/s (hardware) | 2566.22 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,788.31 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.777e-05 | loss scale: 1 | grad norm: 2.5321 | skipped iterations: 0 | nan iterations: 0 | average step time 28417.29 ms | remaining 1:15:18 | completion 2026-09-08 13:04:54 (24.29 %) | load_balancing_loss: 816.94012 | load_balancing_loss: nan | lm_head_loss: 0.17090 | run: 0 +2026-09-08 11:50:01,931 [Rank 00] training @ step 52/ 210 | consumed tokens: 109,051,904 | batch size: 2097152 | step time: 26077.98 ms | throughput: 64378.86 tflop/s (model) | 79871.10 tflop/s (hardware) | 2513.08 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.58 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.767e-05 | loss scale: 1 | grad norm: 4.3307 | skipped iterations: 0 | nan iterations: 0 | average step time 28372.31 ms | remaining 1:14:43 | completion 2026-09-08 13:04:45 (24.76 %) | load_balancing_loss: 829.67090 | load_balancing_loss: nan | lm_head_loss: 0.18116 | run: 0 +2026-09-08 11:50:30,484 [Rank 00] training @ step 53/ 210 | consumed tokens: 111,149,056 | batch size: 2097152 | step time: 28553.04 ms | throughput: 58798.31 tflop/s (model) | 72947.63 tflop/s (hardware) | 2295.24 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.29 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.757e-05 | loss scale: 1 | grad norm: 45.2344 | skipped iterations: 0 | nan iterations: 0 | average step time 28375.72 ms | remaining 1:14:15 | completion 2026-09-08 13:04:45 (25.24 %) | load_balancing_loss: 833.23914 | load_balancing_loss: nan | lm_head_loss: 0.17131 | run: 0 +2026-09-08 11:50:56,494 [Rank 00] training @ step 54/ 210 | consumed tokens: 113,246,208 | batch size: 2097152 | step time: 26010.81 ms | throughput: 64545.10 tflop/s (model) | 80077.35 tflop/s (hardware) | 2519.57 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.10 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.747e-05 | loss scale: 1 | grad norm: 4.1061 | skipped iterations: 0 | nan iterations: 0 | average step time 28331.92 ms | remaining 1:13:40 | completion 2026-09-08 13:04:36 (25.71 %) | load_balancing_loss: 823.57544 | load_balancing_loss: nan | lm_head_loss: 0.16543 | run: 0 +2026-09-08 11:51:23,121 [Rank 00] training @ step 55/ 210 | consumed tokens: 115,343,360 | batch size: 2097152 | step time: 26626.76 ms | throughput: 63052.00 tflop/s (model) | 78224.94 tflop/s (hardware) | 2461.28 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.42 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.737e-05 | loss scale: 1 | grad norm: 13.4025 | skipped iterations: 0 | nan iterations: 0 | average step time 28300.92 ms | remaining 1:13:07 | completion 2026-09-08 13:04:30 (26.19 %) | load_balancing_loss: 814.26105 | load_balancing_loss: nan | lm_head_loss: 0.16151 | run: 0 +2026-09-08 11:51:51,095 [Rank 00] training @ step 56/ 210 | consumed tokens: 117,440,512 | batch size: 2097152 | step time: 27973.54 ms | throughput: 60016.39 tflop/s (model) | 74458.84 tflop/s (hardware) | 2342.79 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,788.83 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.726e-05 | loss scale: 1 | grad norm: 5.1722 | skipped iterations: 0 | nan iterations: 0 | average step time 28295.07 ms | remaining 1:12:37 | completion 2026-09-08 13:04:29 (26.67 %) | load_balancing_loss: 821.15930 | load_balancing_loss: nan | lm_head_loss: 0.16172 | run: 0 +2026-09-08 11:52:17,655 [Rank 00] training @ step 57/ 210 | consumed tokens: 119,537,664 | batch size: 2097152 | step time: 26560.66 ms | throughput: 63208.92 tflop/s (model) | 78419.62 tflop/s (hardware) | 2467.41 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.28 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.715e-05 | loss scale: 1 | grad norm: 2.4420 | skipped iterations: 0 | nan iterations: 0 | average step time 28264.64 ms | remaining 1:12:04 | completion 2026-09-08 13:04:22 (27.14 %) | load_balancing_loss: 817.47284 | load_balancing_loss: nan | lm_head_loss: 0.15669 | run: 0 +2026-09-08 11:52:45,783 [Rank 00] training @ step 58/ 210 | consumed tokens: 121,634,816 | batch size: 2097152 | step time: 28127.39 ms | throughput: 59688.12 tflop/s (model) | 74051.56 tflop/s (hardware) | 2329.97 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,786.99 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.704e-05 | loss scale: 1 | grad norm: 7.1050 | skipped iterations: 0 | nan iterations: 0 | average step time 28262.28 ms | remaining 1:11:36 | completion 2026-09-08 13:04:22 (27.62 %) | load_balancing_loss: 814.93890 | load_balancing_loss: nan | lm_head_loss: 0.17920 | run: 0 +2026-09-08 11:53:12,662 [Rank 00] training @ step 59/ 210 | consumed tokens: 123,731,968 | batch size: 2097152 | step time: 26879.45 ms | throughput: 62459.25 tflop/s (model) | 77489.55 tflop/s (hardware) | 2438.14 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.43 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.693e-05 | loss scale: 1 | grad norm: 1.7812 | skipped iterations: 0 | nan iterations: 0 | average step time 28238.84 ms | remaining 1:11:04 | completion 2026-09-08 13:04:17 (28.10 %) | load_balancing_loss: 812.88367 | load_balancing_loss: nan | lm_head_loss: 0.16691 | run: 0 +2026-09-08 11:53:40,029 [Rank 00] training @ step 60/ 210 | consumed tokens: 125,829,120 | batch size: 2097152 | step time: 27367.16 ms | throughput: 61346.18 tflop/s (model) | 76108.63 tflop/s (hardware) | 2394.70 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.66 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.682e-05 | loss scale: 1 | grad norm: 90.8325 | skipped iterations: 0 | nan iterations: 0 | average step time 28224.31 ms | remaining 1:10:34 | completion 2026-09-08 13:04:14 (28.57 %) | load_balancing_loss: 813.63232 | load_balancing_loss: nan | lm_head_loss: 0.16773 | run: 0 +2026-09-08 11:54:07,440 [Rank 00] training @ step 61/ 210 | consumed tokens: 127,926,272 | batch size: 2097152 | step time: 27410.97 ms | throughput: 61248.12 tflop/s (model) | 75986.97 tflop/s (hardware) | 2390.87 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.90 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.671e-05 | loss scale: 1 | grad norm: 2.2029 | skipped iterations: 0 | nan iterations: 0 | average step time 28210.98 ms | remaining 1:10:03 | completion 2026-09-08 13:04:11 (29.05 %) | load_balancing_loss: 835.97723 | load_balancing_loss: nan | lm_head_loss: 0.14660 | run: 0 +2026-09-08 11:54:33,742 [Rank 00] training @ step 62/ 210 | consumed tokens: 130,023,424 | batch size: 2097152 | step time: 26301.39 ms | throughput: 63832.02 tflop/s (model) | 79192.67 tflop/s (hardware) | 2491.73 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,788.14 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.659e-05 | loss scale: 1 | grad norm: 6.2322 | skipped iterations: 0 | nan iterations: 0 | average step time 28180.18 ms | remaining 1:09:31 | completion 2026-09-08 13:04:04 (29.52 %) | load_balancing_loss: 821.41937 | load_balancing_loss: nan | lm_head_loss: 0.16353 | run: 0 +2026-09-08 11:54:59,997 [Rank 00] training @ step 63/ 210 | consumed tokens: 132,120,576 | batch size: 2097152 | step time: 26255.11 ms | throughput: 63944.53 tflop/s (model) | 79332.25 tflop/s (hardware) | 2496.12 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,788.14 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.647e-05 | loss scale: 1 | grad norm: 1.7423 | skipped iterations: 0 | nan iterations: 0 | average step time 28149.62 ms | remaining 1:08:58 | completion 2026-09-08 13:03:58 (30.00 %) | load_balancing_loss: 819.71320 | load_balancing_loss: nan | lm_head_loss: 0.16570 | run: 0 +2026-09-08 11:55:27,468 [Rank 00] training @ step 64/ 210 | consumed tokens: 134,217,728 | batch size: 2097152 | step time: 27470.97 ms | throughput: 61114.36 tflop/s (model) | 75821.02 tflop/s (hardware) | 2385.65 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.43 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.636e-05 | loss scale: 1 | grad norm: 2.1053 | skipped iterations: 0 | nan iterations: 0 | average step time 28139.02 ms | remaining 1:08:28 | completion 2026-09-08 13:03:56 (30.48 %) | load_balancing_loss: 813.09296 | load_balancing_loss: nan | lm_head_loss: 0.17077 | run: 0 +2026-09-08 11:55:55,670 [Rank 00] training @ step 65/ 210 | consumed tokens: 136,314,880 | batch size: 2097152 | step time: 28202.43 ms | throughput: 59529.30 tflop/s (model) | 73854.53 tflop/s (hardware) | 2323.77 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,788.26 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.623e-05 | loss scale: 1 | grad norm: 4.5885 | skipped iterations: 0 | nan iterations: 0 | average step time 28139.99 ms | remaining 1:08:00 | completion 2026-09-08 13:03:56 (30.95 %) | load_balancing_loss: 818.61719 | load_balancing_loss: nan | lm_head_loss: 0.15730 | run: 0 +2026-09-08 11:56:23,226 [Rank 00] training @ step 66/ 210 | consumed tokens: 138,412,032 | batch size: 2097152 | step time: 27555.55 ms | throughput: 60926.77 tflop/s (model) | 75588.29 tflop/s (hardware) | 2378.32 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.56 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.611e-05 | loss scale: 1 | grad norm: 1.5094 | skipped iterations: 0 | nan iterations: 0 | average step time 28131.14 ms | remaining 1:07:31 | completion 2026-09-08 13:03:54 (31.43 %) | load_balancing_loss: 804.77167 | load_balancing_loss: nan | lm_head_loss: 0.16430 | run: 0 +2026-09-08 11:56:51,533 [Rank 00] training @ step 67/ 210 | consumed tokens: 140,509,184 | batch size: 2097152 | step time: 28307.35 ms | throughput: 59308.66 tflop/s (model) | 73580.79 tflop/s (hardware) | 2315.16 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.66 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.599e-05 | loss scale: 1 | grad norm: 4.0308 | skipped iterations: 0 | nan iterations: 0 | average step time 28133.77 ms | remaining 1:07:03 | completion 2026-09-08 13:03:55 (31.90 %) | load_balancing_loss: 816.84033 | load_balancing_loss: nan | lm_head_loss: 0.15134 | run: 0 +2026-09-08 11:57:21,246 [Rank 00] training @ step 68/ 210 | consumed tokens: 142,606,336 | batch size: 2097152 | step time: 29712.65 ms | throughput: 56503.56 tflop/s (model) | 70100.67 tflop/s (hardware) | 2205.66 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.35 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.587e-05 | loss scale: 1 | grad norm: 5.9080 | skipped iterations: 0 | nan iterations: 0 | average step time 28156.99 ms | remaining 1:06:38 | completion 2026-09-08 13:04:00 (32.38 %) | load_balancing_loss: 828.01984 | load_balancing_loss: nan | lm_head_loss: 0.15148 | run: 0 +2026-09-08 11:57:48,580 [Rank 00] training @ step 69/ 210 | consumed tokens: 144,703,488 | batch size: 2097152 | step time: 27334.12 ms | throughput: 61420.33 tflop/s (model) | 76200.62 tflop/s (hardware) | 2397.59 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.49 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.574e-05 | loss scale: 1 | grad norm: 3.2443 | skipped iterations: 0 | nan iterations: 0 | average step time 28145.06 ms | remaining 1:06:08 | completion 2026-09-08 13:03:57 (32.86 %) | load_balancing_loss: 820.58673 | load_balancing_loss: nan | lm_head_loss: 0.16142 | run: 0 +2026-09-08 11:58:16,363 [Rank 00] training @ step 70/ 210 | consumed tokens: 146,800,640 | batch size: 2097152 | step time: 27782.79 ms | throughput: 60428.45 tflop/s (model) | 74970.05 tflop/s (hardware) | 2358.87 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,787.26 MiB | reserved 61,408.00 MiB | max reserved 61,408.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.561e-05 | loss scale: 1 | grad norm: 4.3366 | skipped iterations: 0 | nan iterations: 0 | average step time 28139.89 ms | remaining 1:05:40 | completion 2026-09-08 13:03:56 (33.33 %) | load_balancing_loss: 823.45361 | load_balancing_loss: nan | lm_head_loss: 0.15508 | run: 0 +2026-09-08 11:58:16,364 [Rank 00] Saving export at iteration 70 +2026-09-08 11:58:17,238 [Rank 00] Saving tensors to /mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3/export/gemma4/70/model_0.safetensors +2026-09-08 11:59:00,798 [Rank 00] Saving tensors to /mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3/export/gemma4/70/model_1.safetensors +2026-09-08 11:59:42,577 [Rank 00] Saving tensors to /mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3/export/gemma4/70/model_2.safetensors +2026-09-08 12:00:27,122 [Rank 00] Saving tensors to /mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3/export/gemma4/70/model_3.safetensors +2026-09-08 12:01:08,789 [Rank 00] Saving tensors to /mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3/export/gemma4/70/model_4.safetensors +2026-09-08 12:01:52,854 [Rank 00] Saving tensors to /mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3/export/gemma4/70/model_5.safetensors +2026-09-08 12:02:19,838 [Rank 00] Saving index to /mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3/export/gemma4/70/model.safetensors.index.json +2026-09-08 12:02:19,844 [Rank 00] Saved export to /mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3/export/gemma4/70 +2026-09-08 12:02:47,541 [Rank 00] training @ step 71/ 210 | consumed tokens: 148,897,792 | batch size: 2097152 | step time: 271178.39 ms | throughput: 6191.02 tflop/s (model) | 7680.84 tflop/s (hardware) | 241.67 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.77 MiB | reserved 55,560.00 MiB | max reserved 74,782.00 MiB | global max reserved 61,408.00 MiB | learning rate: 1.548e-05 | loss scale: 1 | grad norm: 13.4651 | skipped iterations: 0 | nan iterations: 0 | average step time 31562.96 ms | remaining 1:13:07 | completion 2026-09-08 13:15:55 (33.81 %) | load_balancing_loss: 811.32642 | load_balancing_loss: nan | lm_head_loss: 0.16040 | run: 0 +2026-09-08 12:03:14,359 [Rank 00] training @ step 72/ 210 | consumed tokens: 150,994,944 | batch size: 2097152 | step time: 26817.92 ms | throughput: 62602.56 tflop/s (model) | 77667.35 tflop/s (hardware) | 2443.74 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.84 MiB | reserved 55,560.00 MiB | max reserved 55,560.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.535e-05 | loss scale: 1 | grad norm: 3.5607 | skipped iterations: 0 | nan iterations: 0 | average step time 31497.06 ms | remaining 1:12:27 | completion 2026-09-08 13:15:41 (34.29 %) | load_balancing_loss: 818.17517 | load_balancing_loss: nan | lm_head_loss: 0.16850 | run: 0 +2026-09-08 12:03:42,063 [Rank 00] training @ step 73/ 210 | consumed tokens: 153,092,096 | batch size: 2097152 | step time: 27704.16 ms | throughput: 60599.95 tflop/s (model) | 75182.82 tflop/s (hardware) | 2365.57 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.10 MiB | reserved 55,560.00 MiB | max reserved 55,560.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.522e-05 | loss scale: 1 | grad norm: 6.5278 | skipped iterations: 0 | nan iterations: 0 | average step time 31445.10 ms | remaining 1:11:48 | completion 2026-09-08 13:15:30 (34.76 %) | load_balancing_loss: 808.15503 | load_balancing_loss: nan | lm_head_loss: 0.16601 | run: 0 +2026-09-08 12:04:11,088 [Rank 00] training @ step 74/ 210 | consumed tokens: 155,189,248 | batch size: 2097152 | step time: 29024.61 ms | throughput: 57843.00 tflop/s (model) | 71762.43 tflop/s (hardware) | 2257.95 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.34 MiB | reserved 55,560.00 MiB | max reserved 55,560.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.509e-05 | loss scale: 1 | grad norm: 2.2200 | skipped iterations: 0 | nan iterations: 0 | average step time 31412.39 ms | remaining 1:11:12 | completion 2026-09-08 13:15:23 (35.24 %) | load_balancing_loss: 825.43237 | load_balancing_loss: nan | lm_head_loss: 0.15726 | run: 0 +2026-09-08 12:04:39,101 [Rank 00] training @ step 75/ 210 | consumed tokens: 157,286,400 | batch size: 2097152 | step time: 28013.07 ms | throughput: 59931.70 tflop/s (model) | 74353.76 tflop/s (hardware) | 2339.48 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.38 MiB | reserved 55,560.00 MiB | max reserved 55,560.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.496e-05 | loss scale: 1 | grad norm: 0.9700 | skipped iterations: 0 | nan iterations: 0 | average step time 31367.07 ms | remaining 1:10:35 | completion 2026-09-08 13:15:14 (35.71 %) | load_balancing_loss: 820.60278 | load_balancing_loss: nan | lm_head_loss: 0.14700 | run: 0 +2026-09-08 12:05:07,823 [Rank 00] training @ step 76/ 210 | consumed tokens: 159,383,552 | batch size: 2097152 | step time: 28721.68 ms | throughput: 58453.09 tflop/s (model) | 72519.33 tflop/s (hardware) | 2281.76 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.46 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.482e-05 | loss scale: 1 | grad norm: 1.6724 | skipped iterations: 0 | nan iterations: 0 | average step time 31332.26 ms | remaining 1:09:59 | completion 2026-09-08 13:15:06 (36.19 %) | load_balancing_loss: 823.72107 | load_balancing_loss: nan | lm_head_loss: 0.14002 | run: 0 +2026-09-08 12:05:35,374 [Rank 00] training @ step 77/ 210 | consumed tokens: 161,480,704 | batch size: 2097152 | step time: 27551.23 ms | throughput: 60936.32 tflop/s (model) | 75600.14 tflop/s (hardware) | 2378.70 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.14 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.468e-05 | loss scale: 1 | grad norm: 7.1342 | skipped iterations: 0 | nan iterations: 0 | average step time 31283.16 ms | remaining 1:09:21 | completion 2026-09-08 13:14:56 (36.67 %) | load_balancing_loss: 821.76050 | load_balancing_loss: nan | lm_head_loss: 0.13892 | run: 0 +2026-09-08 12:06:02,833 [Rank 00] training @ step 78/ 210 | consumed tokens: 163,577,856 | batch size: 2097152 | step time: 27459.40 ms | throughput: 61140.11 tflop/s (model) | 75852.97 tflop/s (hardware) | 2386.65 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.79 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.455e-05 | loss scale: 1 | grad norm: 9.7638 | skipped iterations: 0 | nan iterations: 0 | average step time 31234.13 ms | remaining 1:08:43 | completion 2026-09-08 13:14:46 (37.14 %) | load_balancing_loss: 813.84778 | load_balancing_loss: nan | lm_head_loss: 0.13900 | run: 0 +2026-09-08 12:06:29,470 [Rank 00] training @ step 79/ 210 | consumed tokens: 165,675,008 | batch size: 2097152 | step time: 26636.45 ms | throughput: 63029.08 tflop/s (model) | 78196.50 tflop/s (hardware) | 2460.39 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.66 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.441e-05 | loss scale: 1 | grad norm: 61.1826 | skipped iterations: 0 | nan iterations: 0 | average step time 31175.94 ms | remaining 1:08:04 | completion 2026-09-08 13:14:34 (37.62 %) | load_balancing_loss: 821.56317 | load_balancing_loss: nan | lm_head_loss: 0.14618 | run: 0 +2026-09-08 12:06:56,830 [Rank 00] training @ step 80/ 210 | consumed tokens: 167,772,160 | batch size: 2097152 | step time: 27359.84 ms | throughput: 61362.58 tflop/s (model) | 76128.98 tflop/s (hardware) | 2395.34 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.94 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.427e-05 | loss scale: 1 | grad norm: 6.9074 | skipped iterations: 0 | nan iterations: 0 | average step time 31128.24 ms | remaining 1:07:27 | completion 2026-09-08 13:14:23 (38.10 %) | load_balancing_loss: 818.56702 | load_balancing_loss: nan | lm_head_loss: 0.14308 | run: 0 +2026-09-08 12:07:24,727 [Rank 00] training @ step 81/ 210 | consumed tokens: 169,869,312 | batch size: 2097152 | step time: 27897.79 ms | throughput: 60179.35 tflop/s (model) | 74661.01 tflop/s (hardware) | 2349.15 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.08 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.413e-05 | loss scale: 1 | grad norm: 4.9085 | skipped iterations: 0 | nan iterations: 0 | average step time 31088.35 ms | remaining 1:06:50 | completion 2026-09-08 13:14:15 (38.57 %) | load_balancing_loss: 807.68848 | load_balancing_loss: nan | lm_head_loss: 0.14893 | run: 0 +2026-09-08 12:07:53,226 [Rank 00] training @ step 82/ 210 | consumed tokens: 171,966,464 | batch size: 2097152 | step time: 28498.26 ms | throughput: 58911.34 tflop/s (model) | 73087.87 tflop/s (hardware) | 2299.65 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.92 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.399e-05 | loss scale: 1 | grad norm: 4.2847 | skipped iterations: 0 | nan iterations: 0 | average step time 31056.77 ms | remaining 1:06:15 | completion 2026-09-08 13:14:08 (39.05 %) | load_balancing_loss: 818.18469 | load_balancing_loss: nan | lm_head_loss: 0.15433 | run: 0 +2026-09-08 12:08:21,378 [Rank 00] training @ step 83/ 210 | consumed tokens: 174,063,616 | batch size: 2097152 | step time: 28152.29 ms | throughput: 59635.31 tflop/s (model) | 73986.05 tflop/s (hardware) | 2327.91 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.34 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.384e-05 | loss scale: 1 | grad norm: 8.3736 | skipped iterations: 0 | nan iterations: 0 | average step time 31021.77 ms | remaining 1:05:40 | completion 2026-09-08 13:14:01 (39.52 %) | load_balancing_loss: 815.11377 | load_balancing_loss: nan | lm_head_loss: 0.11913 | run: 0 +2026-09-08 12:08:49,013 [Rank 00] training @ step 84/ 210 | consumed tokens: 176,160,768 | batch size: 2097152 | step time: 27635.41 ms | throughput: 60750.71 tflop/s (model) | 75369.87 tflop/s (hardware) | 2371.45 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.26 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.370e-05 | loss scale: 1 | grad norm: 3.1764 | skipped iterations: 0 | nan iterations: 0 | average step time 30981.46 ms | remaining 1:05:04 | completion 2026-09-08 13:13:53 (40.00 %) | load_balancing_loss: 810.12451 | load_balancing_loss: nan | lm_head_loss: 0.15036 | run: 0 +2026-09-08 12:09:17,129 [Rank 00] training @ step 85/ 210 | consumed tokens: 178,257,920 | batch size: 2097152 | step time: 28115.93 ms | throughput: 59712.44 tflop/s (model) | 74081.75 tflop/s (hardware) | 2330.92 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.28 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.356e-05 | loss scale: 1 | grad norm: 5.3911 | skipped iterations: 0 | nan iterations: 0 | average step time 30947.75 ms | remaining 1:04:28 | completion 2026-09-08 13:13:46 (40.48 %) | load_balancing_loss: 811.46838 | load_balancing_loss: nan | lm_head_loss: 0.12865 | run: 0 +2026-09-08 12:09:44,243 [Rank 00] training @ step 86/ 210 | consumed tokens: 180,355,072 | batch size: 2097152 | step time: 27113.86 ms | throughput: 61919.28 tflop/s (model) | 76819.64 tflop/s (hardware) | 2417.07 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.06 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.341e-05 | loss scale: 1 | grad norm: 14.2226 | skipped iterations: 0 | nan iterations: 0 | average step time 30903.17 ms | remaining 1:03:52 | completion 2026-09-08 13:13:36 (40.95 %) | load_balancing_loss: 813.16919 | load_balancing_loss: nan | lm_head_loss: 0.14182 | run: 0 +2026-09-08 12:10:11,892 [Rank 00] training @ step 87/ 210 | consumed tokens: 182,452,224 | batch size: 2097152 | step time: 27648.95 ms | throughput: 60720.95 tflop/s (model) | 75332.94 tflop/s (hardware) | 2370.29 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.53 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.327e-05 | loss scale: 1 | grad norm: 2.2324 | skipped iterations: 0 | nan iterations: 0 | average step time 30865.76 ms | remaining 1:03:16 | completion 2026-09-08 13:13:28 (41.43 %) | load_balancing_loss: 813.16089 | load_balancing_loss: nan | lm_head_loss: 0.13072 | run: 0 +2026-09-08 12:10:39,163 [Rank 00] training @ step 88/ 210 | consumed tokens: 184,549,376 | batch size: 2097152 | step time: 27270.53 ms | throughput: 61563.56 tflop/s (model) | 76378.31 tflop/s (hardware) | 2403.18 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.93 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.312e-05 | loss scale: 1 | grad norm: 2.4558 | skipped iterations: 0 | nan iterations: 0 | average step time 30824.91 ms | remaining 1:02:41 | completion 2026-09-08 13:13:20 (41.90 %) | load_balancing_loss: 811.16302 | load_balancing_loss: nan | lm_head_loss: 0.13967 | run: 0 +2026-09-08 12:11:07,034 [Rank 00] training @ step 89/ 210 | consumed tokens: 186,646,528 | batch size: 2097152 | step time: 27871.52 ms | throughput: 60236.06 tflop/s (model) | 74731.37 tflop/s (hardware) | 2351.36 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.75 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.297e-05 | loss scale: 1 | grad norm: 12.7628 | skipped iterations: 0 | nan iterations: 0 | average step time 30791.72 ms | remaining 1:02:06 | completion 2026-09-08 13:13:13 (42.38 %) | load_balancing_loss: 821.22125 | load_balancing_loss: nan | lm_head_loss: 0.13694 | run: 0 +2026-09-08 12:11:35,578 [Rank 00] training @ step 90/ 210 | consumed tokens: 188,743,680 | batch size: 2097152 | step time: 28543.30 ms | throughput: 58818.39 tflop/s (model) | 72972.54 tflop/s (hardware) | 2296.02 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.61 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.282e-05 | loss scale: 1 | grad norm: 12.8191 | skipped iterations: 0 | nan iterations: 0 | average step time 30766.74 ms | remaining 1:01:32 | completion 2026-09-08 13:13:08 (42.86 %) | load_balancing_loss: 822.86707 | load_balancing_loss: nan | lm_head_loss: 0.13432 | run: 0 +2026-09-08 12:12:02,863 [Rank 00] training @ step 91/ 210 | consumed tokens: 190,840,832 | batch size: 2097152 | step time: 27286.06 ms | throughput: 61528.51 tflop/s (model) | 76334.83 tflop/s (hardware) | 2401.81 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.82 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.268e-05 | loss scale: 1 | grad norm: 6.5536 | skipped iterations: 0 | nan iterations: 0 | average step time 30728.49 ms | remaining 1:00:57 | completion 2026-09-08 13:13:00 (43.33 %) | load_balancing_loss: 811.76727 | load_balancing_loss: nan | lm_head_loss: 0.14872 | run: 0 +2026-09-08 12:12:28,767 [Rank 00] training @ step 92/ 210 | consumed tokens: 192,937,984 | batch size: 2097152 | step time: 25903.87 ms | throughput: 64811.59 tflop/s (model) | 80407.96 tflop/s (hardware) | 2529.97 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.75 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.253e-05 | loss scale: 1 | grad norm: 2.1942 | skipped iterations: 0 | nan iterations: 0 | average step time 30676.05 ms | remaining 1:00:20 | completion 2026-09-08 13:12:49 (43.81 %) | load_balancing_loss: 817.37634 | load_balancing_loss: nan | lm_head_loss: 0.13081 | run: 0 +2026-09-08 12:12:55,993 [Rank 00] training @ step 93/ 210 | consumed tokens: 195,035,136 | batch size: 2097152 | step time: 27226.01 ms | throughput: 61664.23 tflop/s (model) | 76503.21 tflop/s (hardware) | 2407.11 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.82 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.238e-05 | loss scale: 1 | grad norm: 2.0417 | skipped iterations: 0 | nan iterations: 0 | average step time 30638.95 ms | remaining 0:59:45 | completion 2026-09-08 13:12:41 (44.29 %) | load_balancing_loss: 817.99799 | load_balancing_loss: nan | lm_head_loss: 0.12667 | run: 0 +2026-09-08 12:13:22,964 [Rank 00] training @ step 94/ 210 | consumed tokens: 197,132,288 | batch size: 2097152 | step time: 26970.82 ms | throughput: 62247.67 tflop/s (model) | 77227.05 tflop/s (hardware) | 2429.89 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.43 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.223e-05 | loss scale: 1 | grad norm: 2.2028 | skipped iterations: 0 | nan iterations: 0 | average step time 30599.93 ms | remaining 0:59:10 | completion 2026-09-08 13:12:33 (44.76 %) | load_balancing_loss: 831.66742 | load_balancing_loss: nan | lm_head_loss: 0.13087 | run: 0 +2026-09-08 12:13:49,135 [Rank 00] training @ step 95/ 210 | consumed tokens: 199,229,440 | batch size: 2097152 | step time: 26170.38 ms | throughput: 64151.56 tflop/s (model) | 79589.09 tflop/s (hardware) | 2504.21 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.17 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.207e-05 | loss scale: 1 | grad norm: 3.5990 | skipped iterations: 0 | nan iterations: 0 | average step time 30553.30 ms | remaining 0:58:34 | completion 2026-09-08 13:12:23 (45.24 %) | load_balancing_loss: 824.11853 | load_balancing_loss: nan | lm_head_loss: 0.13808 | run: 0 +2026-09-08 12:14:15,791 [Rank 00] training @ step 96/ 210 | consumed tokens: 201,326,592 | batch size: 2097152 | step time: 26656.61 ms | throughput: 62981.41 tflop/s (model) | 78137.36 tflop/s (hardware) | 2458.53 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.78 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.192e-05 | loss scale: 1 | grad norm: 9.4708 | skipped iterations: 0 | nan iterations: 0 | average step time 30512.71 ms | remaining 0:57:58 | completion 2026-09-08 13:12:14 (45.71 %) | load_balancing_loss: 808.43652 | load_balancing_loss: nan | lm_head_loss: 0.12422 | run: 0 +2026-09-08 12:14:43,176 [Rank 00] training @ step 97/ 210 | consumed tokens: 203,423,744 | batch size: 2097152 | step time: 27384.67 ms | throughput: 61306.96 tflop/s (model) | 76059.97 tflop/s (hardware) | 2393.16 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.80 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.177e-05 | loss scale: 1 | grad norm: 2.6236 | skipped iterations: 0 | nan iterations: 0 | average step time 30480.46 ms | remaining 0:57:24 | completion 2026-09-08 13:12:07 (46.19 %) | load_balancing_loss: 816.58063 | load_balancing_loss: nan | lm_head_loss: 0.14583 | run: 0 +2026-09-08 12:15:11,588 [Rank 00] training @ step 98/ 210 | consumed tokens: 205,520,896 | batch size: 2097152 | step time: 28412.09 ms | throughput: 59090.01 tflop/s (model) | 73309.52 tflop/s (hardware) | 2306.62 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.04 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.162e-05 | loss scale: 1 | grad norm: 3.9726 | skipped iterations: 0 | nan iterations: 0 | average step time 30459.36 ms | remaining 0:56:51 | completion 2026-09-08 13:12:03 (46.67 %) | load_balancing_loss: 828.95221 | load_balancing_loss: nan | lm_head_loss: 0.12991 | run: 0 +2026-09-08 12:15:40,228 [Rank 00] training @ step 99/ 210 | consumed tokens: 207,618,048 | batch size: 2097152 | step time: 28639.70 ms | throughput: 58620.40 tflop/s (model) | 72726.91 tflop/s (hardware) | 2288.29 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.79 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.146e-05 | loss scale: 1 | grad norm: 2.7306 | skipped iterations: 0 | nan iterations: 0 | average step time 30440.98 ms | remaining 0:56:19 | completion 2026-09-08 13:11:59 (47.14 %) | load_balancing_loss: 816.66779 | load_balancing_loss: nan | lm_head_loss: 0.12918 | run: 0 +2026-09-08 12:16:07,973 [Rank 00] training @ step 100/ 210 | consumed tokens: 209,715,200 | batch size: 2097152 | step time: 27745.35 ms | throughput: 60509.99 tflop/s (model) | 75071.21 tflop/s (hardware) | 2362.05 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.73 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.131e-05 | loss scale: 1 | grad norm: 5.5473 | skipped iterations: 0 | nan iterations: 0 | average step time 30414.02 ms | remaining 0:55:46 | completion 2026-09-08 13:11:54 (47.62 %) | load_balancing_loss: 815.95050 | load_balancing_loss: nan | lm_head_loss: 0.13854 | run: 0 +2026-09-08 12:16:34,204 [Rank 00] training @ step 101/ 210 | consumed tokens: 211,812,352 | batch size: 2097152 | step time: 26230.66 ms | throughput: 64004.13 tflop/s (model) | 79406.19 tflop/s (hardware) | 2498.45 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.26 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.116e-05 | loss scale: 1 | grad norm: 18.3715 | skipped iterations: 0 | nan iterations: 0 | average step time 30372.60 ms | remaining 0:55:11 | completion 2026-09-08 13:11:45 (48.10 %) | load_balancing_loss: 804.38989 | load_balancing_loss: nan | lm_head_loss: 0.14125 | run: 0 +2026-09-08 12:17:02,810 [Rank 00] training @ step 102/ 210 | consumed tokens: 213,909,504 | batch size: 2097152 | step time: 28606.05 ms | throughput: 58689.35 tflop/s (model) | 72812.46 tflop/s (hardware) | 2290.98 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.22 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.100e-05 | loss scale: 1 | grad norm: 3.3824 | skipped iterations: 0 | nan iterations: 0 | average step time 30355.28 ms | remaining 0:54:38 | completion 2026-09-08 13:11:41 (48.57 %) | load_balancing_loss: 821.65076 | load_balancing_loss: nan | lm_head_loss: 0.12700 | run: 0 +2026-09-08 12:17:29,728 [Rank 00] training @ step 103/ 210 | consumed tokens: 216,006,656 | batch size: 2097152 | step time: 26918.35 ms | throughput: 62369.01 tflop/s (model) | 77377.59 tflop/s (hardware) | 2434.62 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.96 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.085e-05 | loss scale: 1 | grad norm: 2.8746 | skipped iterations: 0 | nan iterations: 0 | average step time 30321.92 ms | remaining 0:54:04 | completion 2026-09-08 13:11:34 (49.05 %) | load_balancing_loss: 800.26562 | load_balancing_loss: nan | lm_head_loss: 0.13950 | run: 0 +2026-09-08 12:17:58,436 [Rank 00] training @ step 104/ 210 | consumed tokens: 218,103,808 | batch size: 2097152 | step time: 28707.40 ms | throughput: 58482.16 tflop/s (model) | 72555.40 tflop/s (hardware) | 2282.90 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.49 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.070e-05 | loss scale: 1 | grad norm: 2.1634 | skipped iterations: 0 | nan iterations: 0 | average step time 30306.39 ms | remaining 0:53:32 | completion 2026-09-08 13:11:31 (49.52 %) | load_balancing_loss: 820.39685 | load_balancing_loss: nan | lm_head_loss: 0.12731 | run: 0 +2026-09-08 12:18:27,212 [Rank 00] training @ step 105/ 210 | consumed tokens: 220,200,960 | batch size: 2097152 | step time: 28776.53 ms | throughput: 58341.67 tflop/s (model) | 72381.10 tflop/s (hardware) | 2277.41 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.72 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.054e-05 | loss scale: 1 | grad norm: 3.8726 | skipped iterations: 0 | nan iterations: 0 | average step time 30291.82 ms | remaining 0:53:01 | completion 2026-09-08 13:11:28 (50.00 %) | load_balancing_loss: 809.62128 | load_balancing_loss: nan | lm_head_loss: 0.14273 | run: 0 +2026-09-08 12:18:53,905 [Rank 00] training @ step 106/ 210 | consumed tokens: 222,298,112 | batch size: 2097152 | step time: 26692.93 ms | throughput: 62895.71 tflop/s (model) | 78031.04 tflop/s (hardware) | 2455.18 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.25 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.039e-05 | loss scale: 1 | grad norm: 2.9431 | skipped iterations: 0 | nan iterations: 0 | average step time 30257.87 ms | remaining 0:52:27 | completion 2026-09-08 13:11:21 (50.48 %) | load_balancing_loss: 808.65161 | load_balancing_loss: nan | lm_head_loss: 0.12661 | run: 0 +2026-09-08 12:19:22,134 [Rank 00] training @ step 107/ 210 | consumed tokens: 224,395,264 | batch size: 2097152 | step time: 28229.05 ms | throughput: 59473.16 tflop/s (model) | 73784.88 tflop/s (hardware) | 2321.58 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.40 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.023e-05 | loss scale: 1 | grad norm: 2.2764 | skipped iterations: 0 | nan iterations: 0 | average step time 30238.91 ms | remaining 0:51:55 | completion 2026-09-08 13:11:17 (50.95 %) | load_balancing_loss: 824.87024 | load_balancing_loss: nan | lm_head_loss: 0.13330 | run: 0 +2026-09-08 12:19:50,964 [Rank 00] training @ step 108/ 210 | consumed tokens: 226,492,416 | batch size: 2097152 | step time: 28829.79 ms | throughput: 58233.88 tflop/s (model) | 72247.38 tflop/s (hardware) | 2273.20 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.40 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.008e-05 | loss scale: 1 | grad norm: 5.3208 | skipped iterations: 0 | nan iterations: 0 | average step time 30225.86 ms | remaining 0:51:23 | completion 2026-09-08 13:11:14 (51.43 %) | load_balancing_loss: 815.64056 | load_balancing_loss: nan | lm_head_loss: 0.12887 | run: 0 +2026-09-08 12:20:18,642 [Rank 00] training @ step 109/ 210 | consumed tokens: 228,589,568 | batch size: 2097152 | step time: 27678.58 ms | throughput: 60655.96 tflop/s (model) | 75252.31 tflop/s (hardware) | 2367.75 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.89 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 9.923e-06 | loss scale: 1 | grad norm: 5.5231 | skipped iterations: 0 | nan iterations: 0 | average step time 30202.49 ms | remaining 0:50:50 | completion 2026-09-08 13:11:09 (51.90 %) | load_balancing_loss: 815.52460 | load_balancing_loss: nan | lm_head_loss: 0.13224 | run: 0 +2026-09-08 12:20:44,309 [Rank 00] training @ step 110/ 210 | consumed tokens: 230,686,720 | batch size: 2097152 | step time: 25666.30 ms | throughput: 65411.48 tflop/s (model) | 81152.21 tflop/s (hardware) | 2553.39 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.14 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 9.768e-06 | loss scale: 1 | grad norm: 2.7488 | skipped iterations: 0 | nan iterations: 0 | average step time 30161.25 ms | remaining 0:50:16 | completion 2026-09-08 13:11:00 (52.38 %) | load_balancing_loss: 821.48883 | load_balancing_loss: nan | lm_head_loss: 0.13983 | run: 0 +2026-09-08 12:21:13,102 [Rank 00] training @ step 111/ 210 | consumed tokens: 232,783,872 | batch size: 2097152 | step time: 28792.92 ms | throughput: 58308.46 tflop/s (model) | 72339.90 tflop/s (hardware) | 2276.12 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.01 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 9.613e-06 | loss scale: 1 | grad norm: 3.7493 | skipped iterations: 0 | nan iterations: 0 | average step time 30148.93 ms | remaining 0:49:45 | completion 2026-09-08 13:10:58 (52.86 %) | load_balancing_loss: 804.13513 | load_balancing_loss: nan | lm_head_loss: 0.14333 | run: 0 +2026-09-08 12:21:41,370 [Rank 00] training @ step 112/ 210 | consumed tokens: 234,881,024 | batch size: 2097152 | step time: 28268.04 ms | throughput: 59391.13 tflop/s (model) | 73683.11 tflop/s (hardware) | 2318.38 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.81 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 9.459e-06 | loss scale: 1 | grad norm: 22.2534 | skipped iterations: 0 | nan iterations: 0 | average step time 30132.13 ms | remaining 0:49:13 | completion 2026-09-08 13:10:54 (53.33 %) | load_balancing_loss: 827.05310 | load_balancing_loss: nan | lm_head_loss: 0.13156 | run: 0 +2026-09-08 12:22:10,239 [Rank 00] training @ step 113/ 210 | consumed tokens: 236,978,176 | batch size: 2097152 | step time: 28868.88 ms | throughput: 58155.04 tflop/s (model) | 72149.57 tflop/s (hardware) | 2270.13 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.65 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 9.304e-06 | loss scale: 1 | grad norm: 3.1792 | skipped iterations: 0 | nan iterations: 0 | average step time 30120.95 ms | remaining 0:48:42 | completion 2026-09-08 13:10:52 (53.81 %) | load_balancing_loss: 812.58704 | load_balancing_loss: nan | lm_head_loss: 0.12285 | run: 0 +2026-09-08 12:22:37,905 [Rank 00] training @ step 114/ 210 | consumed tokens: 239,075,328 | batch size: 2097152 | step time: 27666.23 ms | throughput: 60683.04 tflop/s (model) | 75285.90 tflop/s (hardware) | 2368.81 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.54 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 9.150e-06 | loss scale: 1 | grad norm: 8.9435 | skipped iterations: 0 | nan iterations: 0 | average step time 30099.42 ms | remaining 0:48:10 | completion 2026-09-08 13:10:47 (54.29 %) | load_balancing_loss: 817.88617 | load_balancing_loss: nan | lm_head_loss: 0.12781 | run: 0 +2026-09-08 12:23:05,347 [Rank 00] training @ step 115/ 210 | consumed tokens: 241,172,480 | batch size: 2097152 | step time: 27442.43 ms | throughput: 61177.92 tflop/s (model) | 75899.87 tflop/s (hardware) | 2388.13 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.99 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 8.996e-06 | loss scale: 1 | grad norm: 9.0649 | skipped iterations: 0 | nan iterations: 0 | average step time 30076.32 ms | remaining 0:47:37 | completion 2026-09-08 13:10:43 (54.76 %) | load_balancing_loss: 836.41858 | load_balancing_loss: nan | lm_head_loss: 0.13342 | run: 0 +2026-09-08 12:23:32,860 [Rank 00] training @ step 116/ 210 | consumed tokens: 243,269,632 | batch size: 2097152 | step time: 27512.30 ms | throughput: 61022.56 tflop/s (model) | 75707.13 tflop/s (hardware) | 2382.06 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.59 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 8.842e-06 | loss scale: 1 | grad norm: 5.7647 | skipped iterations: 0 | nan iterations: 0 | average step time 30054.21 ms | remaining 0:47:05 | completion 2026-09-08 13:10:38 (55.24 %) | load_balancing_loss: 824.23407 | load_balancing_loss: nan | lm_head_loss: 0.12651 | run: 0 +2026-09-08 12:24:01,935 [Rank 00] training @ step 117/ 210 | consumed tokens: 245,366,784 | batch size: 2097152 | step time: 29075.89 ms | throughput: 57741.00 tflop/s (model) | 71635.89 tflop/s (hardware) | 2253.96 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.10 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 8.688e-06 | loss scale: 1 | grad norm: 10.8707 | skipped iterations: 0 | nan iterations: 0 | average step time 30045.85 ms | remaining 0:46:34 | completion 2026-09-08 13:10:36 (55.71 %) | load_balancing_loss: 810.94208 | load_balancing_loss: nan | lm_head_loss: 0.13031 | run: 0 +2026-09-08 12:24:29,860 [Rank 00] training @ step 118/ 210 | consumed tokens: 247,463,936 | batch size: 2097152 | step time: 27924.98 ms | throughput: 60120.75 tflop/s (model) | 74588.31 tflop/s (hardware) | 2346.86 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,801.06 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 8.535e-06 | loss scale: 1 | grad norm: 38.4536 | skipped iterations: 0 | nan iterations: 0 | average step time 30027.88 ms | remaining 0:46:03 | completion 2026-09-08 13:10:32 (56.19 %) | load_balancing_loss: 813.16071 | load_balancing_loss: nan | lm_head_loss: 0.13310 | run: 0 +2026-09-08 12:24:56,723 [Rank 00] training @ step 119/ 210 | consumed tokens: 249,561,088 | batch size: 2097152 | step time: 26862.84 ms | throughput: 62497.90 tflop/s (model) | 77537.49 tflop/s (hardware) | 2439.65 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.96 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 8.382e-06 | loss scale: 1 | grad norm: 4.1480 | skipped iterations: 0 | nan iterations: 0 | average step time 30001.28 ms | remaining 0:45:30 | completion 2026-09-08 13:10:27 (56.67 %) | load_balancing_loss: 814.18994 | load_balancing_loss: nan | lm_head_loss: 0.12533 | run: 0 +2026-09-08 12:25:25,149 [Rank 00] training @ step 120/ 210 | consumed tokens: 251,658,240 | batch size: 2097152 | step time: 28425.69 ms | throughput: 59061.74 tflop/s (model) | 73274.45 tflop/s (hardware) | 2305.52 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.22 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 8.230e-06 | loss scale: 1 | grad norm: 7.0708 | skipped iterations: 0 | nan iterations: 0 | average step time 29988.15 ms | remaining 0:44:59 | completion 2026-09-08 13:10:24 (57.14 %) | load_balancing_loss: 819.02490 | load_balancing_loss: nan | lm_head_loss: 0.14758 | run: 0 +2026-09-08 12:25:50,975 [Rank 00] training @ step 121/ 210 | consumed tokens: 253,755,392 | batch size: 2097152 | step time: 25826.57 ms | throughput: 65005.55 tflop/s (model) | 80648.60 tflop/s (hardware) | 2537.54 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.61 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 8.078e-06 | loss scale: 1 | grad norm: 3.0336 | skipped iterations: 0 | nan iterations: 0 | average step time 29953.76 ms | remaining 0:44:26 | completion 2026-09-08 13:10:17 (57.62 %) | load_balancing_loss: 824.13928 | load_balancing_loss: nan | lm_head_loss: 0.12223 | run: 0 +2026-09-08 12:26:17,600 [Rank 00] training @ step 122/ 210 | consumed tokens: 255,852,544 | batch size: 2097152 | step time: 26624.08 ms | throughput: 63058.36 tflop/s (model) | 78232.83 tflop/s (hardware) | 2461.53 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.46 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 7.926e-06 | loss scale: 1 | grad norm: 2.7582 | skipped iterations: 0 | nan iterations: 0 | average step time 29926.46 ms | remaining 0:43:54 | completion 2026-09-08 13:10:11 (58.10 %) | load_balancing_loss: 814.56799 | load_balancing_loss: nan | lm_head_loss: 0.13603 | run: 0 +2026-09-08 12:26:44,844 [Rank 00] training @ step 123/ 210 | consumed tokens: 257,949,696 | batch size: 2097152 | step time: 27244.24 ms | throughput: 61622.97 tflop/s (model) | 76452.02 tflop/s (hardware) | 2405.50 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.47 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 7.775e-06 | loss scale: 1 | grad norm: 52.3518 | skipped iterations: 0 | nan iterations: 0 | average step time 29904.66 ms | remaining 0:43:22 | completion 2026-09-08 13:10:07 (58.57 %) | load_balancing_loss: 825.25012 | load_balancing_loss: nan | lm_head_loss: 0.14243 | run: 0 +2026-09-08 12:27:12,094 [Rank 00] training @ step 124/ 210 | consumed tokens: 260,046,848 | batch size: 2097152 | step time: 27250.73 ms | throughput: 61608.29 tflop/s (model) | 76433.81 tflop/s (hardware) | 2404.93 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.27 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 7.624e-06 | loss scale: 1 | grad norm: 12.9381 | skipped iterations: 0 | nan iterations: 0 | average step time 29883.25 ms | remaining 0:42:50 | completion 2026-09-08 13:10:02 (59.05 %) | load_balancing_loss: 810.23975 | load_balancing_loss: nan | lm_head_loss: 0.13788 | run: 0 +2026-09-08 12:27:14,243 [Rank 00] Sample 7936 from dataset gemma_replay_and_automation_bench) took 2,145.69 ms to load +2026-09-08 12:27:16,454 [Rank 00] Sample 7940 from dataset gemma_replay_and_automation_bench) took 2,210.07 ms to load +2026-09-08 12:27:20,709 [Rank 00] Sample 7944 from dataset gemma_replay_and_automation_bench) took 4,252.70 ms to load +2026-09-08 12:27:27,380 [Rank 00] Sample 7952 from dataset gemma_replay_and_automation_bench) took 6,643.99 ms to load +2026-09-08 12:27:33,831 [Rank 00] Sample 7956 from dataset gemma_replay_and_automation_bench) took 6,449.46 ms to load +2026-09-08 12:27:39,752 [Rank 00] Sample 7960 from dataset gemma_replay_and_automation_bench) took 5,919.16 ms to load +2026-09-08 12:27:40,952 [Rank 00] Sample 7964 from dataset gemma_replay_and_automation_bench) took 1,067.88 ms to load +2026-09-08 12:27:43,137 [Rank 00] Sample 7968 from dataset gemma_replay_and_automation_bench) took 2,183.56 ms to load +2026-09-08 12:27:45,920 [Rank 00] Sample 7972 from dataset gemma_replay_and_automation_bench) took 2,781.54 ms to load +2026-09-08 12:27:48,488 [Rank 00] Sample 7984 from dataset gemma_replay_and_automation_bench) took 1,750.66 ms to load +2026-09-08 12:27:50,459 [Rank 00] Sample 7988 from dataset gemma_replay_and_automation_bench) took 1,970.02 ms to load +2026-09-08 12:27:55,221 [Rank 00] Sample 7992 from dataset gemma_replay_and_automation_bench) took 4,760.50 ms to load +2026-09-08 12:27:57,593 [Rank 00] Sample 7996 from dataset gemma_replay_and_automation_bench) took 2,370.27 ms to load +2026-09-08 12:27:57,595 [Rank 00] Data loading took 45,498.39 ms +2026-09-08 12:28:24,501 [Rank 00] training @ step 125/ 210 | consumed tokens: 262,144,000 | batch size: 2097152 | step time: 72406.84 ms | throughput: 23186.63 tflop/s (model) | 28766.30 tflop/s (hardware) | 905.11 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.40 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 7.474e-06 | loss scale: 1 | grad norm: 6.6595 | skipped iterations: 0 | nan iterations: 0 | average step time 30223.44 ms | remaining 0:42:49 | completion 2026-09-08 13:11:13 (59.52 %) | load_balancing_loss: 821.30865 | load_balancing_loss: nan | lm_head_loss: 0.13338 | run: 0 +2026-09-08 12:28:29,380 [Rank 00] Sample 8004 from dataset gemma_replay_and_automation_bench) took 4,847.20 ms to load +2026-09-08 12:28:31,209 [Rank 00] Sample 8020 from dataset gemma_replay_and_automation_bench) took 1,071.73 ms to load +2026-09-08 12:28:35,358 [Rank 00] Sample 8044 from dataset gemma_replay_and_automation_bench) took 2,507.84 ms to load +2026-09-08 12:28:37,249 [Rank 00] Sample 8052 from dataset gemma_replay_and_automation_bench) took 1,837.78 ms to load +2026-09-08 12:28:37,862 [Rank 00] Data loading took 13,358.74 ms +2026-09-08 12:29:04,575 [Rank 00] training @ step 126/ 210 | consumed tokens: 264,241,152 | batch size: 2097152 | step time: 40073.24 ms | throughput: 41895.06 tflop/s (model) | 51976.75 tflop/s (hardware) | 1635.41 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.94 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 7.325e-06 | loss scale: 1 | grad norm: 3.9553 | skipped iterations: 0 | nan iterations: 0 | average step time 30301.62 ms | remaining 0:42:25 | completion 2026-09-08 13:11:30 (60.00 %) | load_balancing_loss: 816.33167 | load_balancing_loss: nan | lm_head_loss: 0.13928 | run: 0 +2026-09-08 12:29:31,686 [Rank 00] training @ step 127/ 210 | consumed tokens: 266,338,304 | batch size: 2097152 | step time: 27111.59 ms | throughput: 61924.46 tflop/s (model) | 76826.07 tflop/s (hardware) | 2417.27 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.09 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 7.176e-06 | loss scale: 1 | grad norm: 3.5664 | skipped iterations: 0 | nan iterations: 0 | average step time 30276.50 ms | remaining 0:41:53 | completion 2026-09-08 13:11:25 (60.48 %) | load_balancing_loss: 802.51263 | load_balancing_loss: nan | lm_head_loss: 0.15103 | run: 0 +2026-09-08 12:29:58,842 [Rank 00] training @ step 128/ 210 | consumed tokens: 268,435,456 | batch size: 2097152 | step time: 27155.44 ms | throughput: 61824.47 tflop/s (model) | 76702.01 tflop/s (hardware) | 2413.37 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.27 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 7.028e-06 | loss scale: 1 | grad norm: 2.1214 | skipped iterations: 0 | nan iterations: 0 | average step time 30252.12 ms | remaining 0:41:21 | completion 2026-09-08 13:11:20 (60.95 %) | load_balancing_loss: 809.56104 | load_balancing_loss: nan | lm_head_loss: 0.13249 | run: 0 +2026-09-08 12:30:27,081 [Rank 00] training @ step 129/ 210 | consumed tokens: 270,532,608 | batch size: 2097152 | step time: 28239.56 ms | throughput: 59451.02 tflop/s (model) | 73757.41 tflop/s (hardware) | 2320.72 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.39 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 6.880e-06 | loss scale: 1 | grad norm: 3.9139 | skipped iterations: 0 | nan iterations: 0 | average step time 30236.51 ms | remaining 0:40:49 | completion 2026-09-08 13:11:16 (61.43 %) | load_balancing_loss: 818.05402 | load_balancing_loss: nan | lm_head_loss: 0.12980 | run: 0 +2026-09-08 12:30:56,095 [Rank 00] training @ step 130/ 210 | consumed tokens: 272,629,760 | batch size: 2097152 | step time: 29013.55 ms | throughput: 57865.06 tflop/s (model) | 71789.80 tflop/s (hardware) | 2258.81 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.87 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 6.734e-06 | loss scale: 1 | grad norm: 56.8328 | skipped iterations: 0 | nan iterations: 0 | average step time 30227.11 ms | remaining 0:40:18 | completion 2026-09-08 13:11:14 (61.90 %) | load_balancing_loss: 824.38458 | load_balancing_loss: nan | lm_head_loss: 0.12166 | run: 0 +2026-09-08 12:31:24,215 [Rank 00] training @ step 131/ 210 | consumed tokens: 274,726,912 | batch size: 2097152 | step time: 28120.18 ms | throughput: 59703.41 tflop/s (model) | 74070.54 tflop/s (hardware) | 2330.57 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.64 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 6.588e-06 | loss scale: 1 | grad norm: 16.2380 | skipped iterations: 0 | nan iterations: 0 | average step time 30211.02 ms | remaining 0:39:47 | completion 2026-09-08 13:11:11 (62.38 %) | load_balancing_loss: 789.98663 | load_balancing_loss: nan | lm_head_loss: 0.12055 | run: 0 +2026-09-08 12:31:52,567 [Rank 00] training @ step 132/ 210 | consumed tokens: 276,824,064 | batch size: 2097152 | step time: 28351.84 ms | throughput: 59215.58 tflop/s (model) | 73465.31 tflop/s (hardware) | 2311.53 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.01 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 6.443e-06 | loss scale: 1 | grad norm: 7.1304 | skipped iterations: 0 | nan iterations: 0 | average step time 30196.94 ms | remaining 0:39:15 | completion 2026-09-08 13:11:08 (62.86 %) | load_balancing_loss: 794.33936 | load_balancing_loss: nan | lm_head_loss: 0.13074 | run: 0 +2026-09-08 12:32:18,978 [Rank 00] training @ step 133/ 210 | consumed tokens: 278,921,216 | batch size: 2097152 | step time: 26411.34 ms | throughput: 63566.28 tflop/s (model) | 78862.97 tflop/s (hardware) | 2481.36 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.68 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 6.299e-06 | loss scale: 1 | grad norm: 5.5049 | skipped iterations: 0 | nan iterations: 0 | average step time 30168.48 ms | remaining 0:38:43 | completion 2026-09-08 13:11:02 (63.33 %) | load_balancing_loss: 813.17950 | load_balancing_loss: nan | lm_head_loss: 0.13138 | run: 0 +2026-09-08 12:32:46,374 [Rank 00] training @ step 134/ 210 | consumed tokens: 281,018,368 | batch size: 2097152 | step time: 27395.53 ms | throughput: 61282.66 tflop/s (model) | 76029.82 tflop/s (hardware) | 2392.22 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.48 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 6.155e-06 | loss scale: 1 | grad norm: 10.5896 | skipped iterations: 0 | nan iterations: 0 | average step time 30147.78 ms | remaining 0:38:11 | completion 2026-09-08 13:10:58 (63.81 %) | load_balancing_loss: 813.47418 | load_balancing_loss: nan | lm_head_loss: 0.12927 | run: 0 +2026-09-08 12:33:14,737 [Rank 00] training @ step 135/ 210 | consumed tokens: 283,115,520 | batch size: 2097152 | step time: 28363.75 ms | throughput: 59190.72 tflop/s (model) | 73434.48 tflop/s (hardware) | 2310.56 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.98 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 6.013e-06 | loss scale: 1 | grad norm: 3.4514 | skipped iterations: 0 | nan iterations: 0 | average step time 30134.57 ms | remaining 0:37:40 | completion 2026-09-08 13:10:55 (64.29 %) | load_balancing_loss: 808.28064 | load_balancing_loss: nan | lm_head_loss: 0.12405 | run: 0 +2026-09-08 12:33:42,100 [Rank 00] training @ step 136/ 210 | consumed tokens: 285,212,672 | batch size: 2097152 | step time: 27362.58 ms | throughput: 61356.46 tflop/s (model) | 76121.38 tflop/s (hardware) | 2395.10 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.70 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 5.871e-06 | loss scale: 1 | grad norm: 18.5852 | skipped iterations: 0 | nan iterations: 0 | average step time 30114.18 ms | remaining 0:37:08 | completion 2026-09-08 13:10:51 (64.76 %) | load_balancing_loss: 817.78394 | load_balancing_loss: nan | lm_head_loss: 0.13817 | run: 0 +2026-09-08 12:34:07,788 [Rank 00] training @ step 137/ 210 | consumed tokens: 287,309,824 | batch size: 2097152 | step time: 25687.82 ms | throughput: 65356.68 tflop/s (model) | 81084.22 tflop/s (hardware) | 2551.25 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.29 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 5.731e-06 | loss scale: 1 | grad norm: 65.0809 | skipped iterations: 0 | nan iterations: 0 | average step time 30081.88 ms | remaining 0:36:36 | completion 2026-09-08 13:10:44 (65.24 %) | load_balancing_loss: 811.30835 | load_balancing_loss: nan | lm_head_loss: 0.12645 | run: 0 +2026-09-08 12:34:34,523 [Rank 00] training @ step 138/ 210 | consumed tokens: 289,406,976 | batch size: 2097152 | step time: 26735.52 ms | throughput: 62795.52 tflop/s (model) | 77906.73 tflop/s (hardware) | 2451.27 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.78 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 5.592e-06 | loss scale: 1 | grad norm: 2.1398 | skipped iterations: 0 | nan iterations: 0 | average step time 30057.63 ms | remaining 0:36:04 | completion 2026-09-08 13:10:39 (65.71 %) | load_balancing_loss: 827.05493 | load_balancing_loss: nan | lm_head_loss: 0.13549 | run: 0 +2026-09-08 12:35:02,050 [Rank 00] training @ step 139/ 210 | consumed tokens: 291,504,128 | batch size: 2097152 | step time: 27527.08 ms | throughput: 60989.79 tflop/s (model) | 75666.48 tflop/s (hardware) | 2380.78 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.98 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 5.453e-06 | loss scale: 1 | grad norm: 4.4311 | skipped iterations: 0 | nan iterations: 0 | average step time 30039.42 ms | remaining 0:35:33 | completion 2026-09-08 13:10:35 (66.19 %) | load_balancing_loss: 829.30042 | load_balancing_loss: nan | lm_head_loss: 0.12342 | run: 0 +2026-09-08 12:35:29,040 [Rank 00] training @ step 140/ 210 | consumed tokens: 293,601,280 | batch size: 2097152 | step time: 26989.18 ms | throughput: 62205.33 tflop/s (model) | 77174.52 tflop/s (hardware) | 2428.23 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.86 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 5.316e-06 | loss scale: 1 | grad norm: 2.5688 | skipped iterations: 0 | nan iterations: 0 | average step time 30017.63 ms | remaining 0:35:01 | completion 2026-09-08 13:10:30 (66.67 %) | load_balancing_loss: 828.73389 | load_balancing_loss: nan | lm_head_loss: 0.11953 | run: 0 +2026-09-08 12:35:29,041 [Rank 00] Saving export at iteration 140 +2026-09-08 12:35:29,078 [Rank 00] Saving tensors to /mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3/export/gemma4/140/model_0.safetensors +2026-09-08 12:36:13,022 [Rank 00] Saving tensors to /mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3/export/gemma4/140/model_1.safetensors +2026-09-08 12:36:54,999 [Rank 00] Saving tensors to /mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3/export/gemma4/140/model_2.safetensors +2026-09-08 12:37:40,154 [Rank 00] Saving tensors to /mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3/export/gemma4/140/model_3.safetensors +2026-09-08 12:38:21,811 [Rank 00] Saving tensors to /mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3/export/gemma4/140/model_4.safetensors +2026-09-08 12:39:06,972 [Rank 00] Saving tensors to /mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3/export/gemma4/140/model_5.safetensors +2026-09-08 12:39:34,012 [Rank 00] Saving index to /mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3/export/gemma4/140/model.safetensors.index.json +2026-09-08 12:39:34,020 [Rank 00] Saved export to /mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3/export/gemma4/140 +2026-09-08 12:40:01,444 [Rank 00] training @ step 141/ 210 | consumed tokens: 295,698,432 | batch size: 2097152 | step time: 272404.85 ms | throughput: 6163.15 tflop/s (model) | 7646.26 tflop/s (hardware) | 240.58 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.74 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 5.180e-06 | loss scale: 1 | grad norm: 3.1150 | skipped iterations: 0 | nan iterations: 0 | average step time 31736.69 ms | remaining 0:36:30 | completion 2026-09-08 13:16:31 (67.14 %) | load_balancing_loss: 822.11755 | load_balancing_loss: nan | lm_head_loss: 0.12814 | run: 0 +2026-09-08 12:40:29,637 [Rank 00] training @ step 142/ 210 | consumed tokens: 297,795,584 | batch size: 2097152 | step time: 28192.70 ms | throughput: 59549.83 tflop/s (model) | 73880.00 tflop/s (hardware) | 2324.57 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.51 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 5.045e-06 | loss scale: 1 | grad norm: 6.5315 | skipped iterations: 0 | nan iterations: 0 | average step time 31711.73 ms | remaining 0:35:56 | completion 2026-09-08 13:16:26 (67.62 %) | load_balancing_loss: 806.88391 | load_balancing_loss: nan | lm_head_loss: 0.13205 | run: 0 +2026-09-08 12:40:56,575 [Rank 00] training @ step 143/ 210 | consumed tokens: 299,892,736 | batch size: 2097152 | step time: 26937.55 ms | throughput: 62324.54 tflop/s (model) | 77322.42 tflop/s (hardware) | 2432.89 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.06 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 4.911e-06 | loss scale: 1 | grad norm: 3.7812 | skipped iterations: 0 | nan iterations: 0 | average step time 31678.35 ms | remaining 0:35:22 | completion 2026-09-08 13:16:19 (68.10 %) | load_balancing_loss: 812.29120 | load_balancing_loss: nan | lm_head_loss: 0.13031 | run: 0 +2026-09-08 12:41:23,304 [Rank 00] training @ step 144/ 210 | consumed tokens: 301,989,888 | batch size: 2097152 | step time: 26729.43 ms | throughput: 62809.81 tflop/s (model) | 77924.47 tflop/s (hardware) | 2451.83 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.77 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 4.778e-06 | loss scale: 1 | grad norm: 1.1189 | skipped iterations: 0 | nan iterations: 0 | average step time 31643.98 ms | remaining 0:34:49 | completion 2026-09-08 13:16:12 (68.57 %) | load_balancing_loss: 825.61163 | load_balancing_loss: nan | lm_head_loss: 0.12196 | run: 0 +2026-09-08 12:41:50,018 [Rank 00] training @ step 145/ 210 | consumed tokens: 304,087,040 | batch size: 2097152 | step time: 26714.26 ms | throughput: 62845.49 tflop/s (model) | 77968.74 tflop/s (hardware) | 2453.22 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.29 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 4.647e-06 | loss scale: 1 | grad norm: 3.3837 | skipped iterations: 0 | nan iterations: 0 | average step time 31609.98 ms | remaining 0:34:15 | completion 2026-09-08 13:16:05 (69.05 %) | load_balancing_loss: 820.39703 | load_balancing_loss: nan | lm_head_loss: 0.11738 | run: 0 +2026-09-08 12:42:17,158 [Rank 00] training @ step 146/ 210 | consumed tokens: 306,184,192 | batch size: 2097152 | step time: 27139.96 ms | throughput: 61859.73 tflop/s (model) | 76745.76 tflop/s (hardware) | 2414.74 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.51 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 4.517e-06 | loss scale: 1 | grad norm: 3.5667 | skipped iterations: 0 | nan iterations: 0 | average step time 31579.37 ms | remaining 0:33:41 | completion 2026-09-08 13:15:58 (69.52 %) | load_balancing_loss: 813.91431 | load_balancing_loss: nan | lm_head_loss: 0.12975 | run: 0 +2026-09-08 12:42:44,581 [Rank 00] training @ step 147/ 210 | consumed tokens: 308,281,344 | batch size: 2097152 | step time: 27423.29 ms | throughput: 61220.63 tflop/s (model) | 75952.86 tflop/s (hardware) | 2389.79 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.23 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 4.388e-06 | loss scale: 1 | grad norm: 4.1641 | skipped iterations: 0 | nan iterations: 0 | average step time 31551.09 ms | remaining 0:33:08 | completion 2026-09-08 13:15:52 (70.00 %) | load_balancing_loss: 828.95038 | load_balancing_loss: nan | lm_head_loss: 0.12807 | run: 0 +2026-09-08 12:43:11,886 [Rank 00] training @ step 148/ 210 | consumed tokens: 310,378,496 | batch size: 2097152 | step time: 27304.56 ms | throughput: 61486.82 tflop/s (model) | 76283.11 tflop/s (hardware) | 2400.18 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.90 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 4.261e-06 | loss scale: 1 | grad norm: 10.4008 | skipped iterations: 0 | nan iterations: 0 | average step time 31522.40 ms | remaining 0:32:34 | completion 2026-09-08 13:15:46 (70.48 %) | load_balancing_loss: 811.50714 | load_balancing_loss: nan | lm_head_loss: 0.13417 | run: 0 +2026-09-08 12:43:38,319 [Rank 00] training @ step 149/ 210 | consumed tokens: 312,475,648 | batch size: 2097152 | step time: 26432.97 ms | throughput: 63514.26 tflop/s (model) | 78798.43 tflop/s (hardware) | 2479.33 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.79 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 4.135e-06 | loss scale: 1 | grad norm: 1.9576 | skipped iterations: 0 | nan iterations: 0 | average step time 31488.24 ms | remaining 0:32:01 | completion 2026-09-08 13:15:39 (70.95 %) | load_balancing_loss: 839.34564 | load_balancing_loss: nan | lm_head_loss: 0.11890 | run: 0 +2026-09-08 12:44:03,943 [Rank 00] training @ step 150/ 210 | consumed tokens: 314,572,800 | batch size: 2097152 | step time: 25624.33 ms | throughput: 65518.61 tflop/s (model) | 81285.12 tflop/s (hardware) | 2557.57 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.98 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 4.010e-06 | loss scale: 1 | grad norm: 10.0866 | skipped iterations: 0 | nan iterations: 0 | average step time 31449.15 ms | remaining 0:31:27 | completion 2026-09-08 13:15:31 (71.43 %) | load_balancing_loss: 830.76190 | load_balancing_loss: nan | lm_head_loss: 0.11327 | run: 0 +2026-09-08 12:44:31,218 [Rank 00] training @ step 151/ 210 | consumed tokens: 316,669,952 | batch size: 2097152 | step time: 27274.40 ms | throughput: 61554.82 tflop/s (model) | 76367.48 tflop/s (hardware) | 2402.84 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.24 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 3.887e-06 | loss scale: 1 | grad norm: 3.5484 | skipped iterations: 0 | nan iterations: 0 | average step time 31421.50 ms | remaining 0:30:54 | completion 2026-09-08 13:15:25 (71.90 %) | load_balancing_loss: 824.73700 | load_balancing_loss: nan | lm_head_loss: 0.12140 | run: 0 +2026-09-08 12:44:58,762 [Rank 00] training @ step 152/ 210 | consumed tokens: 318,767,104 | batch size: 2097152 | step time: 27543.99 ms | throughput: 60952.34 tflop/s (model) | 75620.01 tflop/s (hardware) | 2379.32 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.49 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 3.765e-06 | loss scale: 1 | grad norm: 4.1008 | skipped iterations: 0 | nan iterations: 0 | average step time 31395.99 ms | remaining 0:30:21 | completion 2026-09-08 13:15:20 (72.38 %) | load_balancing_loss: 821.90076 | load_balancing_loss: nan | lm_head_loss: 0.11107 | run: 0 +2026-09-08 12:45:26,766 [Rank 00] training @ step 153/ 210 | consumed tokens: 320,864,256 | batch size: 2097152 | step time: 28004.56 ms | throughput: 59949.89 tflop/s (model) | 74376.34 tflop/s (hardware) | 2340.19 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.29 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 3.645e-06 | loss scale: 1 | grad norm: 4.0966 | skipped iterations: 0 | nan iterations: 0 | average step time 31373.83 ms | remaining 0:29:48 | completion 2026-09-08 13:15:15 (72.86 %) | load_balancing_loss: 806.72345 | load_balancing_loss: nan | lm_head_loss: 0.11197 | run: 0 +2026-09-08 12:45:52,699 [Rank 00] training @ step 154/ 210 | consumed tokens: 322,961,408 | batch size: 2097152 | step time: 25932.83 ms | throughput: 64739.21 tflop/s (model) | 80318.16 tflop/s (hardware) | 2527.14 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.01 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 3.526e-06 | loss scale: 1 | grad norm: 1.2098 | skipped iterations: 0 | nan iterations: 0 | average step time 31338.50 ms | remaining 0:29:15 | completion 2026-09-08 13:15:08 (73.33 %) | load_balancing_loss: 823.53271 | load_balancing_loss: nan | lm_head_loss: 0.10459 | run: 0 +2026-09-08 12:46:21,180 [Rank 00] training @ step 155/ 210 | consumed tokens: 325,058,560 | batch size: 2097152 | step time: 28481.34 ms | throughput: 58946.33 tflop/s (model) | 73131.27 tflop/s (hardware) | 2301.01 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.87 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 3.409e-06 | loss scale: 1 | grad norm: 3.5072 | skipped iterations: 0 | nan iterations: 0 | average step time 31320.06 ms | remaining 0:28:43 | completion 2026-09-08 13:15:04 (73.81 %) | load_balancing_loss: 820.84460 | load_balancing_loss: nan | lm_head_loss: 0.11804 | run: 0 +2026-09-08 12:46:48,246 [Rank 00] training @ step 156/ 210 | consumed tokens: 327,155,712 | batch size: 2097152 | step time: 27065.24 ms | throughput: 62030.50 tflop/s (model) | 76957.63 tflop/s (hardware) | 2421.41 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.72 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 3.293e-06 | loss scale: 1 | grad norm: 2.0000 | skipped iterations: 0 | nan iterations: 0 | average step time 31292.79 ms | remaining 0:28:10 | completion 2026-09-08 13:14:58 (74.29 %) | load_balancing_loss: 807.70435 | load_balancing_loss: nan | lm_head_loss: 0.10611 | run: 0 +2026-09-08 12:47:15,084 [Rank 00] training @ step 157/ 210 | consumed tokens: 329,252,864 | batch size: 2097152 | step time: 26838.80 ms | throughput: 62553.88 tflop/s (model) | 77606.94 tflop/s (hardware) | 2441.84 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.53 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 3.179e-06 | loss scale: 1 | grad norm: 2.0516 | skipped iterations: 0 | nan iterations: 0 | average step time 31264.42 ms | remaining 0:27:37 | completion 2026-09-08 13:14:52 (74.76 %) | load_balancing_loss: 824.99719 | load_balancing_loss: nan | lm_head_loss: 0.10753 | run: 0 +2026-09-08 12:47:41,515 [Rank 00] training @ step 158/ 210 | consumed tokens: 331,350,016 | batch size: 2097152 | step time: 26430.81 ms | throughput: 63519.45 tflop/s (model) | 78804.87 tflop/s (hardware) | 2479.53 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.04 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 3.067e-06 | loss scale: 1 | grad norm: 10.7029 | skipped iterations: 0 | nan iterations: 0 | average step time 31233.83 ms | remaining 0:27:04 | completion 2026-09-08 13:14:46 (75.24 %) | load_balancing_loss: 827.62146 | load_balancing_loss: nan | lm_head_loss: 0.11241 | run: 0 +2026-09-08 12:48:08,578 [Rank 00] training @ step 159/ 210 | consumed tokens: 333,447,168 | batch size: 2097152 | step time: 27062.69 ms | throughput: 62036.37 tflop/s (model) | 76964.90 tflop/s (hardware) | 2421.64 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.29 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 2.956e-06 | loss scale: 1 | grad norm: 7.3379 | skipped iterations: 0 | nan iterations: 0 | average step time 31207.59 ms | remaining 0:26:32 | completion 2026-09-08 13:14:40 (75.71 %) | load_balancing_loss: 808.26410 | load_balancing_loss: nan | lm_head_loss: 0.11560 | run: 0 +2026-09-08 12:48:35,865 [Rank 00] training @ step 160/ 210 | consumed tokens: 335,544,320 | batch size: 2097152 | step time: 27287.16 ms | throughput: 61526.02 tflop/s (model) | 76331.75 tflop/s (hardware) | 2401.72 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.92 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 2.847e-06 | loss scale: 1 | grad norm: 2.7418 | skipped iterations: 0 | nan iterations: 0 | average step time 31183.09 ms | remaining 0:25:59 | completion 2026-09-08 13:14:35 (76.19 %) | load_balancing_loss: 823.07428 | load_balancing_loss: nan | lm_head_loss: 0.10692 | run: 0 +2026-09-08 12:49:03,623 [Rank 00] training @ step 161/ 210 | consumed tokens: 337,641,472 | batch size: 2097152 | step time: 27757.74 ms | throughput: 60482.97 tflop/s (model) | 75037.70 tflop/s (hardware) | 2361.00 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.76 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 2.740e-06 | loss scale: 1 | grad norm: 6.4270 | skipped iterations: 0 | nan iterations: 0 | average step time 31161.81 ms | remaining 0:25:27 | completion 2026-09-08 13:14:31 (76.67 %) | load_balancing_loss: 811.23853 | load_balancing_loss: nan | lm_head_loss: 0.11836 | run: 0 +2026-09-08 12:49:31,356 [Rank 00] training @ step 162/ 210 | consumed tokens: 339,738,624 | batch size: 2097152 | step time: 27733.13 ms | throughput: 60536.65 tflop/s (model) | 75104.29 tflop/s (hardware) | 2363.09 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.93 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 2.634e-06 | loss scale: 1 | grad norm: 4.9554 | skipped iterations: 0 | nan iterations: 0 | average step time 31140.65 ms | remaining 0:24:55 | completion 2026-09-08 13:14:26 (77.14 %) | load_balancing_loss: 805.32050 | load_balancing_loss: nan | lm_head_loss: 0.11476 | run: 0 +2026-09-08 12:49:59,030 [Rank 00] training @ step 163/ 210 | consumed tokens: 341,835,776 | batch size: 2097152 | step time: 27673.99 ms | throughput: 60666.02 tflop/s (model) | 75264.79 tflop/s (hardware) | 2368.14 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.63 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 2.531e-06 | loss scale: 1 | grad norm: 1.8973 | skipped iterations: 0 | nan iterations: 0 | average step time 31119.38 ms | remaining 0:24:23 | completion 2026-09-08 13:14:22 (77.62 %) | load_balancing_loss: 820.69446 | load_balancing_loss: nan | lm_head_loss: 0.10872 | run: 0 +2026-09-08 12:50:26,920 [Rank 00] training @ step 164/ 210 | consumed tokens: 343,932,928 | batch size: 2097152 | step time: 27889.88 ms | throughput: 60196.41 tflop/s (model) | 74682.17 tflop/s (hardware) | 2349.81 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.68 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 2.429e-06 | loss scale: 1 | grad norm: 4.7949 | skipped iterations: 0 | nan iterations: 0 | average step time 31099.69 ms | remaining 0:23:51 | completion 2026-09-08 13:14:18 (78.10 %) | load_balancing_loss: 829.08130 | load_balancing_loss: nan | lm_head_loss: 0.10544 | run: 0 +2026-09-08 12:50:53,947 [Rank 00] training @ step 165/ 210 | consumed tokens: 346,030,080 | batch size: 2097152 | step time: 27026.60 ms | throughput: 62119.21 tflop/s (model) | 77067.67 tflop/s (hardware) | 2424.87 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.39 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 2.329e-06 | loss scale: 1 | grad norm: 6.6353 | skipped iterations: 0 | nan iterations: 0 | average step time 31075.00 ms | remaining 0:23:18 | completion 2026-09-08 13:14:12 (78.57 %) | load_balancing_loss: 824.79578 | load_balancing_loss: nan | lm_head_loss: 0.11804 | run: 0 +2026-09-08 12:51:21,398 [Rank 00] training @ step 166/ 210 | consumed tokens: 348,127,232 | batch size: 2097152 | step time: 27451.58 ms | throughput: 61157.52 tflop/s (model) | 75874.56 tflop/s (hardware) | 2387.33 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.65 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 2.230e-06 | loss scale: 1 | grad norm: 2.1694 | skipped iterations: 0 | nan iterations: 0 | average step time 31053.18 ms | remaining 0:22:46 | completion 2026-09-08 13:14:08 (79.05 %) | load_balancing_loss: 805.85510 | load_balancing_loss: nan | lm_head_loss: 0.12022 | run: 0 +2026-09-08 12:51:50,175 [Rank 00] training @ step 167/ 210 | consumed tokens: 350,224,384 | batch size: 2097152 | step time: 28776.65 ms | throughput: 58341.43 tflop/s (model) | 72380.81 tflop/s (hardware) | 2277.40 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.96 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 2.134e-06 | loss scale: 1 | grad norm: 3.7552 | skipped iterations: 0 | nan iterations: 0 | average step time 31039.54 ms | remaining 0:22:15 | completion 2026-09-08 13:14:05 (79.52 %) | load_balancing_loss: 825.72394 | load_balancing_loss: nan | lm_head_loss: 0.11458 | run: 0 +2026-09-08 12:52:17,264 [Rank 00] training @ step 168/ 210 | consumed tokens: 352,321,536 | batch size: 2097152 | step time: 27089.14 ms | throughput: 61975.77 tflop/s (model) | 76889.73 tflop/s (hardware) | 2419.27 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.77 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 2.039e-06 | loss scale: 1 | grad norm: 5.0681 | skipped iterations: 0 | nan iterations: 0 | average step time 31016.03 ms | remaining 0:21:43 | completion 2026-09-08 13:14:00 (80.00 %) | load_balancing_loss: 816.26172 | load_balancing_loss: nan | lm_head_loss: 0.10509 | run: 0 +2026-09-08 12:52:46,090 [Rank 00] training @ step 169/ 210 | consumed tokens: 354,418,688 | batch size: 2097152 | step time: 28826.09 ms | throughput: 58241.36 tflop/s (model) | 72256.65 tflop/s (hardware) | 2273.50 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.83 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.946e-06 | loss scale: 1 | grad norm: 8.7057 | skipped iterations: 0 | nan iterations: 0 | average step time 31003.07 ms | remaining 0:21:11 | completion 2026-09-08 13:13:57 (80.48 %) | load_balancing_loss: 816.59601 | load_balancing_loss: nan | lm_head_loss: 0.11281 | run: 0 +2026-09-08 12:53:15,178 [Rank 00] training @ step 170/ 210 | consumed tokens: 356,515,840 | batch size: 2097152 | step time: 29088.04 ms | throughput: 57716.88 tflop/s (model) | 71605.96 tflop/s (hardware) | 2253.02 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.46 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.856e-06 | loss scale: 1 | grad norm: 15.4928 | skipped iterations: 0 | nan iterations: 0 | average step time 30991.81 ms | remaining 0:20:40 | completion 2026-09-08 13:13:55 (80.95 %) | load_balancing_loss: 809.45361 | load_balancing_loss: nan | lm_head_loss: 0.10511 | run: 0 +2026-09-08 12:53:44,248 [Rank 00] training @ step 171/ 210 | consumed tokens: 358,612,992 | batch size: 2097152 | step time: 29069.67 ms | throughput: 57753.35 tflop/s (model) | 71651.21 tflop/s (hardware) | 2254.45 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.80 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.767e-06 | loss scale: 1 | grad norm: 1.5324 | skipped iterations: 0 | nan iterations: 0 | average step time 30980.57 ms | remaining 0:20:08 | completion 2026-09-08 13:13:52 (81.43 %) | load_balancing_loss: 815.05945 | load_balancing_loss: nan | lm_head_loss: 0.11226 | run: 0 +2026-09-08 12:54:11,863 [Rank 00] training @ step 172/ 210 | consumed tokens: 360,710,144 | batch size: 2097152 | step time: 27615.70 ms | throughput: 60794.07 tflop/s (model) | 75423.66 tflop/s (hardware) | 2373.14 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.49 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.680e-06 | loss scale: 1 | grad norm: 5.9234 | skipped iterations: 0 | nan iterations: 0 | average step time 30961.00 ms | remaining 0:19:37 | completion 2026-09-08 13:13:48 (81.90 %) | load_balancing_loss: 820.35876 | load_balancing_loss: nan | lm_head_loss: 0.11879 | run: 0 +2026-09-08 12:54:38,933 [Rank 00] training @ step 173/ 210 | consumed tokens: 362,807,296 | batch size: 2097152 | step time: 27070.06 ms | throughput: 62019.47 tflop/s (model) | 76943.93 tflop/s (hardware) | 2420.98 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.79 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.595e-06 | loss scale: 1 | grad norm: 2.8506 | skipped iterations: 0 | nan iterations: 0 | average step time 30938.51 ms | remaining 0:19:05 | completion 2026-09-08 13:13:44 (82.38 %) | load_balancing_loss: 819.79919 | load_balancing_loss: nan | lm_head_loss: 0.11627 | run: 0 +2026-09-08 12:55:05,562 [Rank 00] training @ step 174/ 210 | consumed tokens: 364,904,448 | batch size: 2097152 | step time: 26628.82 ms | throughput: 63047.13 tflop/s (model) | 78218.89 tflop/s (hardware) | 2461.09 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.24 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.512e-06 | loss scale: 1 | grad norm: 2.2613 | skipped iterations: 0 | nan iterations: 0 | average step time 30913.74 ms | remaining 0:18:33 | completion 2026-09-08 13:13:38 (82.86 %) | load_balancing_loss: 811.78223 | load_balancing_loss: nan | lm_head_loss: 0.10767 | run: 0 +2026-09-08 12:55:30,838 [Rank 00] training @ step 175/ 210 | consumed tokens: 367,001,600 | batch size: 2097152 | step time: 25275.68 ms | throughput: 66422.38 tflop/s (model) | 82406.37 tflop/s (hardware) | 2592.85 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.85 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.431e-06 | loss scale: 1 | grad norm: 6.9825 | skipped iterations: 0 | nan iterations: 0 | average step time 30881.53 ms | remaining 0:18:01 | completion 2026-09-08 13:13:32 (83.33 %) | load_balancing_loss: 827.34320 | load_balancing_loss: nan | lm_head_loss: 0.11988 | run: 0 +2026-09-08 12:55:58,442 [Rank 00] training @ step 176/ 210 | consumed tokens: 369,098,752 | batch size: 2097152 | step time: 27603.67 ms | throughput: 60820.57 tflop/s (model) | 75456.53 tflop/s (hardware) | 2374.18 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.94 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.353e-06 | loss scale: 1 | grad norm: 13.9928 | skipped iterations: 0 | nan iterations: 0 | average step time 30862.90 ms | remaining 0:17:29 | completion 2026-09-08 13:13:28 (83.81 %) | load_balancing_loss: 808.86414 | load_balancing_loss: nan | lm_head_loss: 0.11839 | run: 0 +2026-09-08 12:56:25,742 [Rank 00] training @ step 177/ 210 | consumed tokens: 371,195,904 | batch size: 2097152 | step time: 27300.12 ms | throughput: 61496.82 tflop/s (model) | 76295.52 tflop/s (hardware) | 2400.58 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.81 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.276e-06 | loss scale: 1 | grad norm: 2.5819 | skipped iterations: 0 | nan iterations: 0 | average step time 30842.77 ms | remaining 0:16:58 | completion 2026-09-08 13:13:24 (84.29 %) | load_balancing_loss: 820.25452 | load_balancing_loss: nan | lm_head_loss: 0.11053 | run: 0 +2026-09-08 12:56:54,863 [Rank 00] training @ step 178/ 210 | consumed tokens: 373,293,056 | batch size: 2097152 | step time: 29121.37 ms | throughput: 57650.81 tflop/s (model) | 71524.00 tflop/s (hardware) | 2250.44 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.71 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.201e-06 | loss scale: 1 | grad norm: 22.0467 | skipped iterations: 0 | nan iterations: 0 | average step time 30833.10 ms | remaining 0:16:27 | completion 2026-09-08 13:13:22 (84.76 %) | load_balancing_loss: 794.31921 | load_balancing_loss: nan | lm_head_loss: 0.12149 | run: 0 +2026-09-08 12:57:23,634 [Rank 00] training @ step 179/ 210 | consumed tokens: 375,390,208 | batch size: 2097152 | step time: 28770.63 ms | throughput: 58353.64 tflop/s (model) | 72395.96 tflop/s (hardware) | 2277.88 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.07 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.129e-06 | loss scale: 1 | grad norm: 7.6078 | skipped iterations: 0 | nan iterations: 0 | average step time 30821.58 ms | remaining 0:15:55 | completion 2026-09-08 13:13:19 (85.24 %) | load_balancing_loss: 828.95074 | load_balancing_loss: nan | lm_head_loss: 0.11822 | run: 0 +2026-09-08 12:57:49,658 [Rank 00] training @ step 180/ 210 | consumed tokens: 377,487,360 | batch size: 2097152 | step time: 26023.87 ms | throughput: 64512.72 tflop/s (model) | 80037.17 tflop/s (hardware) | 2518.30 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.24 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.059e-06 | loss scale: 1 | grad norm: 5.9900 | skipped iterations: 0 | nan iterations: 0 | average step time 30794.93 ms | remaining 0:15:24 | completion 2026-09-08 13:13:14 (85.71 %) | load_balancing_loss: 822.11572 | load_balancing_loss: nan | lm_head_loss: 0.10733 | run: 0 +2026-09-08 12:58:18,484 [Rank 00] training @ step 181/ 210 | consumed tokens: 379,584,512 | batch size: 2097152 | step time: 28826.35 ms | throughput: 58240.83 tflop/s (model) | 72256.00 tflop/s (hardware) | 2273.48 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.82 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 9.903e-07 | loss scale: 1 | grad norm: 3.8476 | skipped iterations: 0 | nan iterations: 0 | average step time 30784.05 ms | remaining 0:14:53 | completion 2026-09-08 13:13:11 (86.19 %) | load_balancing_loss: 827.63428 | load_balancing_loss: nan | lm_head_loss: 0.10951 | run: 0 +2026-09-08 12:58:46,711 [Rank 00] training @ step 182/ 210 | consumed tokens: 381,681,664 | batch size: 2097152 | step time: 28227.43 ms | throughput: 59476.58 tflop/s (model) | 73789.12 tflop/s (hardware) | 2321.71 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.66 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 9.242e-07 | loss scale: 1 | grad norm: 2.1634 | skipped iterations: 0 | nan iterations: 0 | average step time 30770.00 ms | remaining 0:14:22 | completion 2026-09-08 13:13:08 (86.67 %) | load_balancing_loss: 822.80219 | load_balancing_loss: nan | lm_head_loss: 0.11108 | run: 0 +2026-09-08 12:59:14,020 [Rank 00] training @ step 183/ 210 | consumed tokens: 383,778,816 | batch size: 2097152 | step time: 27308.74 ms | throughput: 61477.41 tflop/s (model) | 76271.43 tflop/s (hardware) | 2399.82 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.51 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 8.604e-07 | loss scale: 1 | grad norm: 6.5673 | skipped iterations: 0 | nan iterations: 0 | average step time 30751.09 ms | remaining 0:13:50 | completion 2026-09-08 13:13:04 (87.14 %) | load_balancing_loss: 812.47363 | load_balancing_loss: nan | lm_head_loss: 0.11396 | run: 0 +2026-09-08 12:59:42,323 [Rank 00] training @ step 184/ 210 | consumed tokens: 385,875,968 | batch size: 2097152 | step time: 28302.98 ms | throughput: 59317.80 tflop/s (model) | 73592.14 tflop/s (hardware) | 2315.52 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.91 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 7.987e-07 | loss scale: 1 | grad norm: 1.9893 | skipped iterations: 0 | nan iterations: 0 | average step time 30737.78 ms | remaining 0:13:19 | completion 2026-09-08 13:13:02 (87.62 %) | load_balancing_loss: 814.63751 | load_balancing_loss: nan | lm_head_loss: 0.11395 | run: 0 +2026-09-08 13:00:10,652 [Rank 00] training @ step 185/ 210 | consumed tokens: 387,973,120 | batch size: 2097152 | step time: 28328.70 ms | throughput: 59263.96 tflop/s (model) | 73525.33 tflop/s (hardware) | 2313.41 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.66 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 7.392e-07 | loss scale: 1 | grad norm: 4.2912 | skipped iterations: 0 | nan iterations: 0 | average step time 30724.76 ms | remaining 0:12:48 | completion 2026-09-08 13:12:59 (88.10 %) | load_balancing_loss: 827.08167 | load_balancing_loss: nan | lm_head_loss: 0.09759 | run: 0 +2026-09-08 13:00:37,981 [Rank 00] training @ step 186/ 210 | consumed tokens: 390,070,272 | batch size: 2097152 | step time: 27328.77 ms | throughput: 61432.36 tflop/s (model) | 76215.55 tflop/s (hardware) | 2398.06 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.97 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 6.819e-07 | loss scale: 1 | grad norm: 2.2949 | skipped iterations: 0 | nan iterations: 0 | average step time 30706.50 ms | remaining 0:12:17 | completion 2026-09-08 13:12:55 (88.57 %) | load_balancing_loss: 824.08447 | load_balancing_loss: nan | lm_head_loss: 0.10809 | run: 0 +2026-09-08 13:01:04,116 [Rank 00] training @ step 187/ 210 | consumed tokens: 392,167,424 | batch size: 2097152 | step time: 26135.32 ms | throughput: 64237.62 tflop/s (model) | 79695.87 tflop/s (hardware) | 2507.56 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,801.14 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 6.268e-07 | loss scale: 1 | grad norm: 5.2956 | skipped iterations: 0 | nan iterations: 0 | average step time 30682.06 ms | remaining 0:11:46 | completion 2026-09-08 13:12:50 (89.05 %) | load_balancing_loss: 826.92932 | load_balancing_loss: nan | lm_head_loss: 0.10941 | run: 0 +2026-09-08 13:01:31,465 [Rank 00] training @ step 188/ 210 | consumed tokens: 394,264,576 | batch size: 2097152 | step time: 27349.11 ms | throughput: 61386.67 tflop/s (model) | 76158.85 tflop/s (hardware) | 2396.28 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,801.08 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 5.740e-07 | loss scale: 1 | grad norm: 40.4846 | skipped iterations: 0 | nan iterations: 0 | average step time 30664.33 ms | remaining 0:11:15 | completion 2026-09-08 13:12:46 (89.52 %) | load_balancing_loss: 812.01709 | load_balancing_loss: nan | lm_head_loss: 0.11380 | run: 0 +2026-09-08 13:01:58,275 [Rank 00] training @ step 189/ 210 | consumed tokens: 396,361,728 | batch size: 2097152 | step time: 26809.92 ms | throughput: 62621.25 tflop/s (model) | 77690.53 tflop/s (hardware) | 2444.47 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.04 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 5.235e-07 | loss scale: 1 | grad norm: 1.5151 | skipped iterations: 0 | nan iterations: 0 | average step time 30643.94 ms | remaining 0:10:44 | completion 2026-09-08 13:12:42 (90.00 %) | load_balancing_loss: 819.55481 | load_balancing_loss: nan | lm_head_loss: 0.10494 | run: 0 +2026-09-08 13:02:25,283 [Rank 00] training @ step 190/ 210 | consumed tokens: 398,458,880 | batch size: 2097152 | step time: 27008.26 ms | throughput: 62161.38 tflop/s (model) | 77119.99 tflop/s (hardware) | 2426.52 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.86 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 4.752e-07 | loss scale: 1 | grad norm: 1.9932 | skipped iterations: 0 | nan iterations: 0 | average step time 30624.80 ms | remaining 0:10:12 | completion 2026-09-08 13:12:38 (90.48 %) | load_balancing_loss: 813.97302 | load_balancing_loss: nan | lm_head_loss: 0.12472 | run: 0 +2026-09-08 13:02:52,153 [Rank 00] training @ step 191/ 210 | consumed tokens: 400,556,032 | batch size: 2097152 | step time: 26869.91 ms | throughput: 62481.45 tflop/s (model) | 77517.09 tflop/s (hardware) | 2439.01 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.51 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 4.292e-07 | loss scale: 1 | grad norm: 17.3734 | skipped iterations: 0 | nan iterations: 0 | average step time 30605.14 ms | remaining 0:09:41 | completion 2026-09-08 13:12:34 (90.95 %) | load_balancing_loss: 831.07977 | load_balancing_loss: nan | lm_head_loss: 0.10650 | run: 0 +2026-09-08 13:03:18,983 [Rank 00] training @ step 192/ 210 | consumed tokens: 402,653,184 | batch size: 2097152 | step time: 26829.56 ms | throughput: 62575.41 tflop/s (model) | 77633.66 tflop/s (hardware) | 2442.68 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.95 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 3.855e-07 | loss scale: 1 | grad norm: 1.6037 | skipped iterations: 0 | nan iterations: 0 | average step time 30585.48 ms | remaining 0:09:11 | completion 2026-09-08 13:12:30 (91.43 %) | load_balancing_loss: 829.68311 | load_balancing_loss: nan | lm_head_loss: 0.11551 | run: 0 +2026-09-08 13:03:45,454 [Rank 00] training @ step 193/ 210 | consumed tokens: 404,750,336 | batch size: 2097152 | step time: 26470.96 ms | throughput: 63423.12 tflop/s (model) | 78685.37 tflop/s (hardware) | 2475.77 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.35 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 3.441e-07 | loss scale: 1 | grad norm: 4.6636 | skipped iterations: 0 | nan iterations: 0 | average step time 30564.16 ms | remaining 0:08:40 | completion 2026-09-08 13:12:25 (91.90 %) | load_balancing_loss: 817.18597 | load_balancing_loss: nan | lm_head_loss: 0.11443 | run: 0 +2026-09-08 13:04:12,787 [Rank 00] training @ step 194/ 210 | consumed tokens: 406,847,488 | batch size: 2097152 | step time: 27333.27 ms | throughput: 61422.24 tflop/s (model) | 76202.99 tflop/s (hardware) | 2397.66 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.79 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 3.050e-07 | loss scale: 1 | grad norm: 36.6506 | skipped iterations: 0 | nan iterations: 0 | average step time 30547.51 ms | remaining 0:08:09 | completion 2026-09-08 13:12:22 (92.38 %) | load_balancing_loss: 820.44214 | load_balancing_loss: nan | lm_head_loss: 0.11045 | run: 0 +2026-09-08 13:04:40,028 [Rank 00] training @ step 195/ 210 | consumed tokens: 408,944,640 | batch size: 2097152 | step time: 27241.18 ms | throughput: 61629.89 tflop/s (model) | 76460.61 tflop/s (hardware) | 2405.77 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.43 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 2.682e-07 | loss scale: 1 | grad norm: 2.5432 | skipped iterations: 0 | nan iterations: 0 | average step time 30530.55 ms | remaining 0:07:38 | completion 2026-09-08 13:12:18 (92.86 %) | load_balancing_loss: 814.82648 | load_balancing_loss: nan | lm_head_loss: 0.11456 | run: 0 +2026-09-08 13:05:07,891 [Rank 00] training @ step 196/ 210 | consumed tokens: 411,041,792 | batch size: 2097152 | step time: 27863.16 ms | throughput: 60254.15 tflop/s (model) | 74753.81 tflop/s (hardware) | 2352.07 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.11 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 2.338e-07 | loss scale: 1 | grad norm: 12.7807 | skipped iterations: 0 | nan iterations: 0 | average step time 30516.94 ms | remaining 0:07:07 | completion 2026-09-08 13:12:15 (93.33 %) | load_balancing_loss: 817.13232 | load_balancing_loss: nan | lm_head_loss: 0.11830 | run: 0 +2026-09-08 13:05:35,802 [Rank 00] training @ step 197/ 210 | consumed tokens: 413,138,944 | batch size: 2097152 | step time: 27910.89 ms | throughput: 60151.09 tflop/s (model) | 74625.95 tflop/s (hardware) | 2348.04 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.04 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 2.017e-07 | loss scale: 1 | grad norm: 5.0515 | skipped iterations: 0 | nan iterations: 0 | average step time 30503.71 ms | remaining 0:06:37 | completion 2026-09-08 13:12:12 (93.81 %) | load_balancing_loss: 820.86737 | load_balancing_loss: nan | lm_head_loss: 0.10225 | run: 0 +2026-09-08 13:06:02,358 [Rank 00] training @ step 198/ 210 | consumed tokens: 415,236,096 | batch size: 2097152 | step time: 26556.44 ms | throughput: 63218.97 tflop/s (model) | 78432.09 tflop/s (hardware) | 2467.80 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.48 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.719e-07 | loss scale: 1 | grad norm: 13.0779 | skipped iterations: 0 | nan iterations: 0 | average step time 30483.78 ms | remaining 0:06:06 | completion 2026-09-08 13:12:08 (94.29 %) | load_balancing_loss: 799.92255 | load_balancing_loss: nan | lm_head_loss: 0.12459 | run: 0 +2026-09-08 13:06:28,797 [Rank 00] training @ step 199/ 210 | consumed tokens: 417,333,248 | batch size: 2097152 | step time: 26438.23 ms | throughput: 63501.62 tflop/s (model) | 78782.76 tflop/s (hardware) | 2478.83 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.79 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.445e-07 | loss scale: 1 | grad norm: 7.7620 | skipped iterations: 0 | nan iterations: 0 | average step time 30463.45 ms | remaining 0:05:35 | completion 2026-09-08 13:12:04 (94.76 %) | load_balancing_loss: 830.13379 | load_balancing_loss: nan | lm_head_loss: 0.10445 | run: 0 +2026-09-08 13:06:56,187 [Rank 00] training @ step 200/ 210 | consumed tokens: 419,430,400 | batch size: 2097152 | step time: 27389.95 ms | throughput: 61295.13 tflop/s (model) | 76045.29 tflop/s (hardware) | 2392.70 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.02 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.195e-07 | loss scale: 1 | grad norm: 1.5548 | skipped iterations: 0 | nan iterations: 0 | average step time 30448.08 ms | remaining 0:05:04 | completion 2026-09-08 13:12:01 (95.24 %) | load_balancing_loss: 821.36237 | load_balancing_loss: nan | lm_head_loss: 0.10991 | run: 0 +2026-09-08 13:07:24,238 [Rank 00] training @ step 201/ 210 | consumed tokens: 421,527,552 | batch size: 2097152 | step time: 28050.90 ms | throughput: 59850.87 tflop/s (model) | 74253.48 tflop/s (hardware) | 2336.32 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.32 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 9.684e-08 | loss scale: 1 | grad norm: 7.1345 | skipped iterations: 0 | nan iterations: 0 | average step time 30436.15 ms | remaining 0:04:34 | completion 2026-09-08 13:11:58 (95.71 %) | load_balancing_loss: 815.47192 | load_balancing_loss: nan | lm_head_loss: 0.10970 | run: 0 +2026-09-08 13:07:52,915 [Rank 00] training @ step 202/ 210 | consumed tokens: 423,624,704 | batch size: 2097152 | step time: 28677.20 ms | throughput: 58543.74 tflop/s (model) | 72631.81 tflop/s (hardware) | 2285.30 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.68 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 7.654e-08 | loss scale: 1 | grad norm: 3.4637 | skipped iterations: 0 | nan iterations: 0 | average step time 30427.45 ms | remaining 0:04:03 | completion 2026-09-08 13:11:56 (96.19 %) | load_balancing_loss: 837.73376 | load_balancing_loss: nan | lm_head_loss: 0.11343 | run: 0 +2026-09-08 13:08:21,118 [Rank 00] training @ step 203/ 210 | consumed tokens: 425,721,856 | batch size: 2097152 | step time: 28203.10 ms | throughput: 59527.89 tflop/s (model) | 73852.78 tflop/s (hardware) | 2323.72 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.89 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 5.862e-08 | loss scale: 1 | grad norm: 1.1900 | skipped iterations: 0 | nan iterations: 0 | average step time 30416.49 ms | remaining 0:03:33 | completion 2026-09-08 13:11:54 (96.67 %) | load_balancing_loss: 819.01868 | load_balancing_loss: nan | lm_head_loss: 0.10077 | run: 0 +2026-09-08 13:08:47,507 [Rank 00] training @ step 204/ 210 | consumed tokens: 427,819,008 | batch size: 2097152 | step time: 26389.13 ms | throughput: 63619.79 tflop/s (model) | 78929.36 tflop/s (hardware) | 2483.45 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.28 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 4.308e-08 | loss scale: 1 | grad norm: 2.5488 | skipped iterations: 0 | nan iterations: 0 | average step time 30396.75 ms | remaining 0:03:02 | completion 2026-09-08 13:11:50 (97.14 %) | load_balancing_loss: 830.85345 | load_balancing_loss: nan | lm_head_loss: 0.10150 | run: 0 +2026-09-08 13:09:14,568 [Rank 00] training @ step 205/ 210 | consumed tokens: 429,916,160 | batch size: 2097152 | step time: 27060.69 ms | throughput: 62040.93 tflop/s (model) | 76970.56 tflop/s (hardware) | 2421.82 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,799.92 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 2.992e-08 | loss scale: 1 | grad norm: 4.6845 | skipped iterations: 0 | nan iterations: 0 | average step time 30380.47 ms | remaining 0:02:32 | completion 2026-09-08 13:11:46 (97.62 %) | load_balancing_loss: 826.56610 | load_balancing_loss: nan | lm_head_loss: 0.10226 | run: 0 +2026-09-08 13:09:41,431 [Rank 00] training @ step 206/ 210 | consumed tokens: 432,013,312 | batch size: 2097152 | step time: 26862.75 ms | throughput: 62498.09 tflop/s (model) | 77537.74 tflop/s (hardware) | 2439.66 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.82 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.915e-08 | loss scale: 1 | grad norm: 2.4464 | skipped iterations: 0 | nan iterations: 0 | average step time 30363.40 ms | remaining 0:02:01 | completion 2026-09-08 13:11:43 (98.10 %) | load_balancing_loss: 817.36279 | load_balancing_loss: nan | lm_head_loss: 0.10863 | run: 0 +2026-09-08 13:10:06,932 [Rank 00] training @ step 207/ 210 | consumed tokens: 434,110,464 | batch size: 2097152 | step time: 25501.13 ms | throughput: 65835.16 tflop/s (model) | 81677.84 tflop/s (hardware) | 2569.93 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.28 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.078e-08 | loss scale: 1 | grad norm: 8.9945 | skipped iterations: 0 | nan iterations: 0 | average step time 30339.91 ms | remaining 0:01:31 | completion 2026-09-08 13:11:38 (98.57 %) | load_balancing_loss: 822.32214 | load_balancing_loss: nan | lm_head_loss: 0.10509 | run: 0 +2026-09-08 13:10:34,277 [Rank 00] training @ step 208/ 210 | consumed tokens: 436,207,616 | batch size: 2097152 | step time: 27345.15 ms | throughput: 61395.56 tflop/s (model) | 76169.89 tflop/s (hardware) | 2396.62 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.06 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 4.790e-09 | loss scale: 1 | grad norm: 5.5213 | skipped iterations: 0 | nan iterations: 0 | average step time 30325.51 ms | remaining 0:01:01 | completion 2026-09-08 13:11:35 (99.05 %) | load_balancing_loss: 833.20758 | load_balancing_loss: nan | lm_head_loss: 0.11608 | run: 0 +2026-09-08 13:11:02,082 [Rank 00] training @ step 209/ 210 | consumed tokens: 438,304,768 | batch size: 2097152 | step time: 27805.10 ms | throughput: 60379.95 tflop/s (model) | 74909.88 tflop/s (hardware) | 2356.98 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.29 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 1.197e-09 | loss scale: 1 | grad norm: 3.4281 | skipped iterations: 0 | nan iterations: 0 | average step time 30313.45 ms | remaining 0:00:30 | completion 2026-09-08 13:11:32 (99.52 %) | load_balancing_loss: 812.37500 | load_balancing_loss: nan | lm_head_loss: 0.11319 | run: 0 +2026-09-08 13:11:30,323 [Rank 00] training @ step 210/ 210 | consumed tokens: 440,401,920 | batch size: 2097152 | step time: 28241.05 ms | throughput: 59447.88 tflop/s (model) | 73753.52 tflop/s (hardware) | 2320.59 tokens/s/gpu | Memory allocated 21,478.50 MiB | max allocated 54,800.11 MiB | reserved 55,562.00 MiB | max reserved 55,562.00 MiB | global max reserved 74,782.00 MiB | learning rate: 0.000e+00 | loss scale: 1 | grad norm: 2.4825 | skipped iterations: 0 | nan iterations: 0 | average step time 30303.58 ms | remaining 0:00:00 | completion 2026-09-08 13:11:30 (100.00 %) | load_balancing_loss: 815.46338 | load_balancing_loss: nan | lm_head_loss: 0.11222 | run: 0 +2026-09-08 13:11:30,324 [Rank 00] Saving export at iteration 210 +2026-09-08 13:11:30,359 [Rank 00] Saving tensors to /mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3/export/gemma4/210/model_0.safetensors +2026-09-08 13:12:14,179 [Rank 00] Saving tensors to /mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3/export/gemma4/210/model_1.safetensors +2026-09-08 13:12:56,239 [Rank 00] Saving tensors to /mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3/export/gemma4/210/model_2.safetensors +2026-09-08 13:13:40,574 [Rank 00] Saving tensors to /mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3/export/gemma4/210/model_3.safetensors +2026-09-08 13:14:22,267 [Rank 00] Saving tensors to /mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3/export/gemma4/210/model_4.safetensors +2026-09-08 13:15:06,657 [Rank 00] Saving tensors to /mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3/export/gemma4/210/model_5.safetensors +2026-09-08 13:15:33,649 [Rank 00] Saving index to /mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3/export/gemma4/210/model.safetensors.index.json +2026-09-08 13:15:33,656 [Rank 00] Saved export to /mnt/queue1/shashank/exp/sft/26_09_08_sft_email_v4_v3/export/gemma4/210 diff --git a/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_01.txt b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_01.txt new file mode 100644 index 0000000000000000000000000000000000000000..0e393560546bcb354c44ce0d0e07cd752d08249e --- /dev/null +++ b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_01.txt @@ -0,0 +1,28 @@ +2026-09-08 11:16:28,778 [Rank 01] >>> Allocating 3 weight buffers (2,975.25 MiB) +2026-09-08 11:16:28,788 [Rank 01] >>> Allocating 3 grad buffers (5,950.49 MiB) +2026-09-08 11:16:28,790 [Rank 01] >>> Allocating 4 shards (12,389.60 MiB) +2026-09-08 11:16:28,793 [Rank 01] Total allocated: 21,315.35 MiB +2026-09-08 11:16:32,858 [Rank 01] Loading index from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model.safetensors.index.json +2026-09-08 11:16:32,863 [Rank 01] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00001-of-00002.safetensors +2026-09-08 11:25:08,562 [Rank 01] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00002-of-00002.safetensors +2026-09-08 11:25:26,457 [Rank 01] 811,965,856 state entries loaded successfully +2026-09-08 11:25:29,458 [Rank 01] Data loading took 1,071.27 ms +2026-09-08 12:27:14,242 [Rank 01] Sample 7936 from dataset gemma_replay_and_automation_bench) took 2,147.60 ms to load +2026-09-08 12:27:16,454 [Rank 01] Sample 7940 from dataset gemma_replay_and_automation_bench) took 1,779.36 ms to load +2026-09-08 12:27:20,709 [Rank 01] Sample 7944 from dataset gemma_replay_and_automation_bench) took 4,253.60 ms to load +2026-09-08 12:27:27,379 [Rank 01] Sample 7952 from dataset gemma_replay_and_automation_bench) took 6,643.74 ms to load +2026-09-08 12:27:33,831 [Rank 01] Sample 7956 from dataset gemma_replay_and_automation_bench) took 6,450.23 ms to load +2026-09-08 12:27:39,751 [Rank 01] Sample 7960 from dataset gemma_replay_and_automation_bench) took 5,919.20 ms to load +2026-09-08 12:27:40,952 [Rank 01] Sample 7964 from dataset gemma_replay_and_automation_bench) took 1,040.60 ms to load +2026-09-08 12:27:43,137 [Rank 01] Sample 7968 from dataset gemma_replay_and_automation_bench) took 2,183.87 ms to load +2026-09-08 12:27:45,920 [Rank 01] Sample 7972 from dataset gemma_replay_and_automation_bench) took 2,782.14 ms to load +2026-09-08 12:27:48,488 [Rank 01] Sample 7984 from dataset gemma_replay_and_automation_bench) took 1,750.62 ms to load +2026-09-08 12:27:50,459 [Rank 01] Sample 7988 from dataset gemma_replay_and_automation_bench) took 1,970.68 ms to load +2026-09-08 12:27:55,221 [Rank 01] Sample 7992 from dataset gemma_replay_and_automation_bench) took 4,760.73 ms to load +2026-09-08 12:27:57,593 [Rank 01] Sample 7996 from dataset gemma_replay_and_automation_bench) took 2,370.77 ms to load +2026-09-08 12:27:57,595 [Rank 01] Data loading took 45,500.24 ms +2026-09-08 12:28:29,380 [Rank 01] Sample 8004 from dataset gemma_replay_and_automation_bench) took 4,847.20 ms to load +2026-09-08 12:28:31,209 [Rank 01] Sample 8020 from dataset gemma_replay_and_automation_bench) took 1,072.00 ms to load +2026-09-08 12:28:35,358 [Rank 01] Sample 8044 from dataset gemma_replay_and_automation_bench) took 2,507.84 ms to load +2026-09-08 12:28:37,249 [Rank 01] Sample 8052 from dataset gemma_replay_and_automation_bench) took 1,837.47 ms to load +2026-09-08 12:28:37,862 [Rank 01] Data loading took 13,360.62 ms diff --git a/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_02.txt b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_02.txt new file mode 100644 index 0000000000000000000000000000000000000000..490ca433208d5255f4110853d1165965d6afc175 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_02.txt @@ -0,0 +1,24 @@ +2026-09-08 11:16:32,846 [Rank 02] Loading index from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model.safetensors.index.json +2026-09-08 11:16:32,852 [Rank 02] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00001-of-00002.safetensors +2026-09-08 11:25:08,562 [Rank 02] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00002-of-00002.safetensors +2026-09-08 11:25:26,457 [Rank 02] 811,965,856 state entries loaded successfully +2026-09-08 11:25:29,441 [Rank 02] Data loading took 1,045.79 ms +2026-09-08 12:27:14,242 [Rank 02] Sample 7936 from dataset gemma_replay_and_automation_bench) took 2,147.69 ms to load +2026-09-08 12:27:16,454 [Rank 02] Sample 7940 from dataset gemma_replay_and_automation_bench) took 2,193.29 ms to load +2026-09-08 12:27:20,709 [Rank 02] Sample 7944 from dataset gemma_replay_and_automation_bench) took 4,253.91 ms to load +2026-09-08 12:27:27,380 [Rank 02] Sample 7952 from dataset gemma_replay_and_automation_bench) took 6,644.59 ms to load +2026-09-08 12:27:33,831 [Rank 02] Sample 7956 from dataset gemma_replay_and_automation_bench) took 6,449.95 ms to load +2026-09-08 12:27:39,752 [Rank 02] Sample 7960 from dataset gemma_replay_and_automation_bench) took 5,919.66 ms to load +2026-09-08 12:27:40,952 [Rank 02] Sample 7964 from dataset gemma_replay_and_automation_bench) took 1,199.07 ms to load +2026-09-08 12:27:43,137 [Rank 02] Sample 7968 from dataset gemma_replay_and_automation_bench) took 2,183.88 ms to load +2026-09-08 12:27:45,920 [Rank 02] Sample 7972 from dataset gemma_replay_and_automation_bench) took 2,781.99 ms to load +2026-09-08 12:27:48,488 [Rank 02] Sample 7984 from dataset gemma_replay_and_automation_bench) took 1,750.79 ms to load +2026-09-08 12:27:50,459 [Rank 02] Sample 7988 from dataset gemma_replay_and_automation_bench) took 1,970.32 ms to load +2026-09-08 12:27:55,221 [Rank 02] Sample 7992 from dataset gemma_replay_and_automation_bench) took 4,760.95 ms to load +2026-09-08 12:27:57,593 [Rank 02] Sample 7996 from dataset gemma_replay_and_automation_bench) took 2,370.71 ms to load +2026-09-08 12:27:57,595 [Rank 02] Data loading took 45,500.21 ms +2026-09-08 12:28:29,380 [Rank 02] Sample 8004 from dataset gemma_replay_and_automation_bench) took 4,847.34 ms to load +2026-09-08 12:28:31,209 [Rank 02] Sample 8020 from dataset gemma_replay_and_automation_bench) took 1,071.66 ms to load +2026-09-08 12:28:35,358 [Rank 02] Sample 8044 from dataset gemma_replay_and_automation_bench) took 2,507.87 ms to load +2026-09-08 12:28:37,249 [Rank 02] Sample 8052 from dataset gemma_replay_and_automation_bench) took 1,837.96 ms to load +2026-09-08 12:28:37,862 [Rank 02] Data loading took 13,360.79 ms diff --git a/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_03.txt b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_03.txt new file mode 100644 index 0000000000000000000000000000000000000000..c9654a12e99fdcf66f00c197ea19525b0c8f738b --- /dev/null +++ b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_03.txt @@ -0,0 +1,24 @@ +2026-09-08 11:16:32,854 [Rank 03] Loading index from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model.safetensors.index.json +2026-09-08 11:16:32,860 [Rank 03] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00001-of-00002.safetensors +2026-09-08 11:25:08,562 [Rank 03] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00002-of-00002.safetensors +2026-09-08 11:25:26,457 [Rank 03] 811,965,856 state entries loaded successfully +2026-09-08 11:25:29,462 [Rank 03] Data loading took 1,058.94 ms +2026-09-08 12:27:14,242 [Rank 03] Sample 7936 from dataset gemma_replay_and_automation_bench) took 2,147.66 ms to load +2026-09-08 12:27:16,454 [Rank 03] Sample 7940 from dataset gemma_replay_and_automation_bench) took 2,194.95 ms to load +2026-09-08 12:27:20,709 [Rank 03] Sample 7944 from dataset gemma_replay_and_automation_bench) took 4,253.30 ms to load +2026-09-08 12:27:27,380 [Rank 03] Sample 7952 from dataset gemma_replay_and_automation_bench) took 6,644.43 ms to load +2026-09-08 12:27:33,831 [Rank 03] Sample 7956 from dataset gemma_replay_and_automation_bench) took 6,450.16 ms to load +2026-09-08 12:27:39,752 [Rank 03] Sample 7960 from dataset gemma_replay_and_automation_bench) took 5,919.29 ms to load +2026-09-08 12:27:40,952 [Rank 03] Sample 7964 from dataset gemma_replay_and_automation_bench) took 1,198.66 ms to load +2026-09-08 12:27:43,137 [Rank 03] Sample 7968 from dataset gemma_replay_and_automation_bench) took 2,184.66 ms to load +2026-09-08 12:27:45,920 [Rank 03] Sample 7972 from dataset gemma_replay_and_automation_bench) took 2,782.05 ms to load +2026-09-08 12:27:48,488 [Rank 03] Sample 7984 from dataset gemma_replay_and_automation_bench) took 1,750.50 ms to load +2026-09-08 12:27:50,459 [Rank 03] Sample 7988 from dataset gemma_replay_and_automation_bench) took 1,969.71 ms to load +2026-09-08 12:27:55,221 [Rank 03] Sample 7992 from dataset gemma_replay_and_automation_bench) took 4,761.11 ms to load +2026-09-08 12:27:57,593 [Rank 03] Sample 7996 from dataset gemma_replay_and_automation_bench) took 2,370.58 ms to load +2026-09-08 12:27:57,594 [Rank 03] Data loading took 45,499.88 ms +2026-09-08 12:28:29,380 [Rank 03] Sample 8004 from dataset gemma_replay_and_automation_bench) took 4,846.43 ms to load +2026-09-08 12:28:31,209 [Rank 03] Sample 8020 from dataset gemma_replay_and_automation_bench) took 1,071.58 ms to load +2026-09-08 12:28:35,358 [Rank 03] Sample 8044 from dataset gemma_replay_and_automation_bench) took 2,508.04 ms to load +2026-09-08 12:28:37,249 [Rank 03] Sample 8052 from dataset gemma_replay_and_automation_bench) took 1,837.60 ms to load +2026-09-08 12:28:37,862 [Rank 03] Data loading took 13,360.79 ms diff --git a/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_04.txt b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_04.txt new file mode 100644 index 0000000000000000000000000000000000000000..576fdc449ff70b8b0917a31f531b834eae60c666 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_04.txt @@ -0,0 +1,24 @@ +2026-09-08 11:16:32,836 [Rank 04] Loading index from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model.safetensors.index.json +2026-09-08 11:16:32,844 [Rank 04] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00001-of-00002.safetensors +2026-09-08 11:25:08,562 [Rank 04] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00002-of-00002.safetensors +2026-09-08 11:25:26,457 [Rank 04] 811,965,856 state entries loaded successfully +2026-09-08 11:25:29,453 [Rank 04] Data loading took 1,063.71 ms +2026-09-08 12:27:14,242 [Rank 04] Sample 7936 from dataset gemma_replay_and_automation_bench) took 2,147.71 ms to load +2026-09-08 12:27:16,454 [Rank 04] Sample 7940 from dataset gemma_replay_and_automation_bench) took 1,774.90 ms to load +2026-09-08 12:27:20,709 [Rank 04] Sample 7944 from dataset gemma_replay_and_automation_bench) took 4,253.32 ms to load +2026-09-08 12:27:27,380 [Rank 04] Sample 7952 from dataset gemma_replay_and_automation_bench) took 6,644.42 ms to load +2026-09-08 12:27:33,831 [Rank 04] Sample 7956 from dataset gemma_replay_and_automation_bench) took 6,450.18 ms to load +2026-09-08 12:27:39,752 [Rank 04] Sample 7960 from dataset gemma_replay_and_automation_bench) took 5,919.21 ms to load +2026-09-08 12:27:40,952 [Rank 04] Sample 7964 from dataset gemma_replay_and_automation_bench) took 1,199.15 ms to load +2026-09-08 12:27:43,137 [Rank 04] Sample 7968 from dataset gemma_replay_and_automation_bench) took 2,183.95 ms to load +2026-09-08 12:27:45,920 [Rank 04] Sample 7972 from dataset gemma_replay_and_automation_bench) took 2,781.73 ms to load +2026-09-08 12:27:48,488 [Rank 04] Sample 7984 from dataset gemma_replay_and_automation_bench) took 1,750.75 ms to load +2026-09-08 12:27:50,459 [Rank 04] Sample 7988 from dataset gemma_replay_and_automation_bench) took 1,969.71 ms to load +2026-09-08 12:27:55,221 [Rank 04] Sample 7992 from dataset gemma_replay_and_automation_bench) took 4,760.97 ms to load +2026-09-08 12:27:57,593 [Rank 04] Sample 7996 from dataset gemma_replay_and_automation_bench) took 2,370.68 ms to load +2026-09-08 12:27:57,595 [Rank 04] Data loading took 45,500.12 ms +2026-09-08 12:28:29,380 [Rank 04] Sample 8004 from dataset gemma_replay_and_automation_bench) took 4,847.40 ms to load +2026-09-08 12:28:31,209 [Rank 04] Sample 8020 from dataset gemma_replay_and_automation_bench) took 1,071.77 ms to load +2026-09-08 12:28:35,358 [Rank 04] Sample 8044 from dataset gemma_replay_and_automation_bench) took 2,507.96 ms to load +2026-09-08 12:28:37,249 [Rank 04] Sample 8052 from dataset gemma_replay_and_automation_bench) took 1,837.63 ms to load +2026-09-08 12:28:37,862 [Rank 04] Data loading took 13,360.85 ms diff --git a/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_05.txt b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_05.txt new file mode 100644 index 0000000000000000000000000000000000000000..4570d17cf222ba591a1f8b3c8c8827e36064588f --- /dev/null +++ b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_05.txt @@ -0,0 +1,24 @@ +2026-09-08 11:16:32,857 [Rank 05] Loading index from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model.safetensors.index.json +2026-09-08 11:16:32,863 [Rank 05] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00001-of-00002.safetensors +2026-09-08 11:25:08,563 [Rank 05] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00002-of-00002.safetensors +2026-09-08 11:25:26,457 [Rank 05] 811,965,856 state entries loaded successfully +2026-09-08 11:25:29,442 [Rank 05] Data loading took 1,055.46 ms +2026-09-08 12:27:14,242 [Rank 05] Sample 7936 from dataset gemma_replay_and_automation_bench) took 2,147.69 ms to load +2026-09-08 12:27:16,454 [Rank 05] Sample 7940 from dataset gemma_replay_and_automation_bench) took 1,715.90 ms to load +2026-09-08 12:27:20,709 [Rank 05] Sample 7944 from dataset gemma_replay_and_automation_bench) took 4,253.40 ms to load +2026-09-08 12:27:27,380 [Rank 05] Sample 7952 from dataset gemma_replay_and_automation_bench) took 6,644.31 ms to load +2026-09-08 12:27:33,831 [Rank 05] Sample 7956 from dataset gemma_replay_and_automation_bench) took 6,450.22 ms to load +2026-09-08 12:27:39,752 [Rank 05] Sample 7960 from dataset gemma_replay_and_automation_bench) took 5,919.21 ms to load +2026-09-08 12:27:40,952 [Rank 05] Sample 7964 from dataset gemma_replay_and_automation_bench) took 1,198.68 ms to load +2026-09-08 12:27:43,137 [Rank 05] Sample 7968 from dataset gemma_replay_and_automation_bench) took 2,184.54 ms to load +2026-09-08 12:27:45,920 [Rank 05] Sample 7972 from dataset gemma_replay_and_automation_bench) took 2,781.97 ms to load +2026-09-08 12:27:48,488 [Rank 05] Sample 7984 from dataset gemma_replay_and_automation_bench) took 1,750.22 ms to load +2026-09-08 12:27:50,459 [Rank 05] Sample 7988 from dataset gemma_replay_and_automation_bench) took 1,969.75 ms to load +2026-09-08 12:27:55,221 [Rank 05] Sample 7992 from dataset gemma_replay_and_automation_bench) took 4,761.19 ms to load +2026-09-08 12:27:57,593 [Rank 05] Sample 7996 from dataset gemma_replay_and_automation_bench) took 2,370.69 ms to load +2026-09-08 12:27:57,595 [Rank 05] Data loading took 45,500.36 ms +2026-09-08 12:28:29,380 [Rank 05] Sample 8004 from dataset gemma_replay_and_automation_bench) took 4,846.40 ms to load +2026-09-08 12:28:31,209 [Rank 05] Sample 8020 from dataset gemma_replay_and_automation_bench) took 1,071.50 ms to load +2026-09-08 12:28:35,358 [Rank 05] Sample 8044 from dataset gemma_replay_and_automation_bench) took 2,508.05 ms to load +2026-09-08 12:28:37,249 [Rank 05] Sample 8052 from dataset gemma_replay_and_automation_bench) took 1,837.89 ms to load +2026-09-08 12:28:37,862 [Rank 05] Data loading took 13,360.76 ms diff --git a/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_06.txt b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_06.txt new file mode 100644 index 0000000000000000000000000000000000000000..ba3c358e441c4e4426e6f1979fa0f5745a735bc0 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_06.txt @@ -0,0 +1,24 @@ +2026-09-08 11:16:32,855 [Rank 06] Loading index from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model.safetensors.index.json +2026-09-08 11:16:32,861 [Rank 06] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00001-of-00002.safetensors +2026-09-08 11:25:08,562 [Rank 06] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00002-of-00002.safetensors +2026-09-08 11:25:26,457 [Rank 06] 811,965,856 state entries loaded successfully +2026-09-08 11:25:29,433 [Rank 06] Data loading took 1,029.95 ms +2026-09-08 12:27:14,242 [Rank 06] Sample 7936 from dataset gemma_replay_and_automation_bench) took 2,147.61 ms to load +2026-09-08 12:27:16,454 [Rank 06] Sample 7940 from dataset gemma_replay_and_automation_bench) took 1,972.11 ms to load +2026-09-08 12:27:20,709 [Rank 06] Sample 7944 from dataset gemma_replay_and_automation_bench) took 4,253.36 ms to load +2026-09-08 12:27:27,380 [Rank 06] Sample 7952 from dataset gemma_replay_and_automation_bench) took 6,644.36 ms to load +2026-09-08 12:27:33,831 [Rank 06] Sample 7956 from dataset gemma_replay_and_automation_bench) took 6,450.18 ms to load +2026-09-08 12:27:39,752 [Rank 06] Sample 7960 from dataset gemma_replay_and_automation_bench) took 5,919.24 ms to load +2026-09-08 12:27:40,952 [Rank 06] Sample 7964 from dataset gemma_replay_and_automation_bench) took 1,076.92 ms to load +2026-09-08 12:27:43,137 [Rank 06] Sample 7968 from dataset gemma_replay_and_automation_bench) took 2,184.62 ms to load +2026-09-08 12:27:45,920 [Rank 06] Sample 7972 from dataset gemma_replay_and_automation_bench) took 2,781.94 ms to load +2026-09-08 12:27:48,488 [Rank 06] Sample 7984 from dataset gemma_replay_and_automation_bench) took 1,750.58 ms to load +2026-09-08 12:27:50,459 [Rank 06] Sample 7988 from dataset gemma_replay_and_automation_bench) took 1,969.70 ms to load +2026-09-08 12:27:55,221 [Rank 06] Sample 7992 from dataset gemma_replay_and_automation_bench) took 4,761.21 ms to load +2026-09-08 12:27:57,593 [Rank 06] Sample 7996 from dataset gemma_replay_and_automation_bench) took 2,370.58 ms to load +2026-09-08 12:27:57,594 [Rank 06] Data loading took 45,499.84 ms +2026-09-08 12:28:29,380 [Rank 06] Sample 8004 from dataset gemma_replay_and_automation_bench) took 4,846.34 ms to load +2026-09-08 12:28:31,209 [Rank 06] Sample 8020 from dataset gemma_replay_and_automation_bench) took 1,071.62 ms to load +2026-09-08 12:28:35,358 [Rank 06] Sample 8044 from dataset gemma_replay_and_automation_bench) took 2,508.13 ms to load +2026-09-08 12:28:37,249 [Rank 06] Sample 8052 from dataset gemma_replay_and_automation_bench) took 1,837.63 ms to load +2026-09-08 12:28:37,862 [Rank 06] Data loading took 13,360.90 ms diff --git a/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_07.txt b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_07.txt new file mode 100644 index 0000000000000000000000000000000000000000..8fde80e3ac75789aff8c03a9b4adef6f952394ab --- /dev/null +++ b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_07.txt @@ -0,0 +1,24 @@ +2026-09-08 11:16:32,859 [Rank 07] Loading index from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model.safetensors.index.json +2026-09-08 11:16:32,866 [Rank 07] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00001-of-00002.safetensors +2026-09-08 11:25:08,562 [Rank 07] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00002-of-00002.safetensors +2026-09-08 11:25:26,457 [Rank 07] 811,965,856 state entries loaded successfully +2026-09-08 11:25:29,442 [Rank 07] Data loading took 1,055.55 ms +2026-09-08 12:27:14,242 [Rank 07] Sample 7936 from dataset gemma_replay_and_automation_bench) took 2,147.70 ms to load +2026-09-08 12:27:16,454 [Rank 07] Sample 7940 from dataset gemma_replay_and_automation_bench) took 1,725.23 ms to load +2026-09-08 12:27:20,709 [Rank 07] Sample 7944 from dataset gemma_replay_and_automation_bench) took 4,253.44 ms to load +2026-09-08 12:27:27,380 [Rank 07] Sample 7952 from dataset gemma_replay_and_automation_bench) took 6,644.05 ms to load +2026-09-08 12:27:33,831 [Rank 07] Sample 7956 from dataset gemma_replay_and_automation_bench) took 6,450.28 ms to load +2026-09-08 12:27:39,752 [Rank 07] Sample 7960 from dataset gemma_replay_and_automation_bench) took 5,919.25 ms to load +2026-09-08 12:27:40,952 [Rank 07] Sample 7964 from dataset gemma_replay_and_automation_bench) took 1,071.54 ms to load +2026-09-08 12:27:43,137 [Rank 07] Sample 7968 from dataset gemma_replay_and_automation_bench) took 2,184.72 ms to load +2026-09-08 12:27:45,920 [Rank 07] Sample 7972 from dataset gemma_replay_and_automation_bench) took 2,782.04 ms to load +2026-09-08 12:27:48,488 [Rank 07] Sample 7984 from dataset gemma_replay_and_automation_bench) took 1,750.23 ms to load +2026-09-08 12:27:50,459 [Rank 07] Sample 7988 from dataset gemma_replay_and_automation_bench) took 1,969.79 ms to load +2026-09-08 12:27:55,221 [Rank 07] Sample 7992 from dataset gemma_replay_and_automation_bench) took 4,761.07 ms to load +2026-09-08 12:27:57,593 [Rank 07] Sample 7996 from dataset gemma_replay_and_automation_bench) took 2,370.37 ms to load +2026-09-08 12:27:57,594 [Rank 07] Data loading took 45,499.79 ms +2026-09-08 12:28:29,380 [Rank 07] Sample 8004 from dataset gemma_replay_and_automation_bench) took 4,846.43 ms to load +2026-09-08 12:28:31,209 [Rank 07] Sample 8020 from dataset gemma_replay_and_automation_bench) took 1,071.47 ms to load +2026-09-08 12:28:35,358 [Rank 07] Sample 8044 from dataset gemma_replay_and_automation_bench) took 2,508.11 ms to load +2026-09-08 12:28:37,249 [Rank 07] Sample 8052 from dataset gemma_replay_and_automation_bench) took 1,837.76 ms to load +2026-09-08 12:28:37,862 [Rank 07] Data loading took 13,360.86 ms diff --git a/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_08.txt b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_08.txt new file mode 100644 index 0000000000000000000000000000000000000000..6172927e8d3f567c3af0612bb2564920c601dd44 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_08.txt @@ -0,0 +1,7 @@ +2026-09-08 11:16:32,901 [Rank 08] Loading index from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model.safetensors.index.json +2026-09-08 11:16:32,908 [Rank 08] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00001-of-00002.safetensors +2026-09-08 11:25:08,563 [Rank 08] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00002-of-00002.safetensors +2026-09-08 11:25:26,458 [Rank 08] 811,965,856 state entries loaded successfully +2026-09-08 11:25:29,467 [Rank 08] Data loading took 1,072.85 ms +2026-09-08 12:27:57,597 [Rank 08] Data loading took 45,502.07 ms +2026-09-08 12:28:37,863 [Rank 08] Data loading took 13,360.93 ms diff --git a/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_09.txt b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_09.txt new file mode 100644 index 0000000000000000000000000000000000000000..e290346b018880c571b4c63b1734f74bbd7cbe5b --- /dev/null +++ b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_09.txt @@ -0,0 +1,7 @@ +2026-09-08 11:16:32,897 [Rank 09] Loading index from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model.safetensors.index.json +2026-09-08 11:16:32,904 [Rank 09] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00001-of-00002.safetensors +2026-09-08 11:25:08,562 [Rank 09] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00002-of-00002.safetensors +2026-09-08 11:25:26,458 [Rank 09] 811,965,856 state entries loaded successfully +2026-09-08 11:25:29,467 [Rank 09] Data loading took 1,070.70 ms +2026-09-08 12:27:57,595 [Rank 09] Data loading took 45,500.69 ms +2026-09-08 12:28:37,863 [Rank 09] Data loading took 13,361.42 ms diff --git a/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_10.txt b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_10.txt new file mode 100644 index 0000000000000000000000000000000000000000..34a4f5f58ef7c281a1733b53078d2156305b8a0a --- /dev/null +++ b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_10.txt @@ -0,0 +1,7 @@ +2026-09-08 11:16:32,907 [Rank 10] Loading index from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model.safetensors.index.json +2026-09-08 11:16:32,914 [Rank 10] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00001-of-00002.safetensors +2026-09-08 11:25:08,563 [Rank 10] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00002-of-00002.safetensors +2026-09-08 11:25:26,458 [Rank 10] 811,965,856 state entries loaded successfully +2026-09-08 11:25:29,449 [Rank 10] Data loading took 1,066.14 ms +2026-09-08 12:27:57,595 [Rank 10] Data loading took 45,500.46 ms +2026-09-08 12:28:37,863 [Rank 10] Data loading took 13,361.03 ms diff --git a/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_11.txt b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_11.txt new file mode 100644 index 0000000000000000000000000000000000000000..8531886181bb80fa9b14db3f6c076e657c6a0c63 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_11.txt @@ -0,0 +1,7 @@ +2026-09-08 11:16:32,872 [Rank 11] Loading index from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model.safetensors.index.json +2026-09-08 11:16:32,880 [Rank 11] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00001-of-00002.safetensors +2026-09-08 11:25:08,562 [Rank 11] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00002-of-00002.safetensors +2026-09-08 11:25:26,458 [Rank 11] 811,965,856 state entries loaded successfully +2026-09-08 11:25:29,470 [Rank 11] Data loading took 1,076.40 ms +2026-09-08 12:27:57,595 [Rank 11] Data loading took 45,500.22 ms +2026-09-08 12:28:37,863 [Rank 11] Data loading took 13,361.11 ms diff --git a/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_12.txt b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_12.txt new file mode 100644 index 0000000000000000000000000000000000000000..79025e3882ce1937c9f1ed73121bb5cf0e946d1b --- /dev/null +++ b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_12.txt @@ -0,0 +1,7 @@ +2026-09-08 11:16:32,906 [Rank 12] Loading index from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model.safetensors.index.json +2026-09-08 11:16:32,912 [Rank 12] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00001-of-00002.safetensors +2026-09-08 11:25:08,563 [Rank 12] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00002-of-00002.safetensors +2026-09-08 11:25:26,458 [Rank 12] 811,965,856 state entries loaded successfully +2026-09-08 11:25:29,463 [Rank 12] Data loading took 1,078.94 ms +2026-09-08 12:27:57,595 [Rank 12] Data loading took 45,500.56 ms +2026-09-08 12:28:37,863 [Rank 12] Data loading took 13,361.22 ms diff --git a/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_13.txt b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_13.txt new file mode 100644 index 0000000000000000000000000000000000000000..63a1a00b0b8245224c3c86460fb89530caf5349f --- /dev/null +++ b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_13.txt @@ -0,0 +1,7 @@ +2026-09-08 11:16:32,899 [Rank 13] Loading index from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model.safetensors.index.json +2026-09-08 11:16:32,905 [Rank 13] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00001-of-00002.safetensors +2026-09-08 11:25:08,563 [Rank 13] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00002-of-00002.safetensors +2026-09-08 11:25:26,458 [Rank 13] 811,965,856 state entries loaded successfully +2026-09-08 11:25:29,454 [Rank 13] Data loading took 1,057.61 ms +2026-09-08 12:27:57,595 [Rank 13] Data loading took 45,500.80 ms +2026-09-08 12:28:37,863 [Rank 13] Data loading took 13,361.08 ms diff --git a/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_14.txt b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_14.txt new file mode 100644 index 0000000000000000000000000000000000000000..48115265a90d2d2245ecf07b30c10b3569edd420 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_14.txt @@ -0,0 +1,7 @@ +2026-09-08 11:16:32,901 [Rank 14] Loading index from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model.safetensors.index.json +2026-09-08 11:16:32,907 [Rank 14] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00001-of-00002.safetensors +2026-09-08 11:25:08,562 [Rank 14] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00002-of-00002.safetensors +2026-09-08 11:25:26,458 [Rank 14] 811,965,856 state entries loaded successfully +2026-09-08 11:25:29,438 [Rank 14] Data loading took 1,057.20 ms +2026-09-08 12:27:57,595 [Rank 14] Data loading took 45,500.32 ms +2026-09-08 12:28:37,863 [Rank 14] Data loading took 13,361.74 ms diff --git a/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_15.txt b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_15.txt new file mode 100644 index 0000000000000000000000000000000000000000..c1797aee606243da082a6bfaa3854fcc47ff266c --- /dev/null +++ b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_15.txt @@ -0,0 +1,7 @@ +2026-09-08 11:16:32,902 [Rank 15] Loading index from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model.safetensors.index.json +2026-09-08 11:16:32,908 [Rank 15] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00001-of-00002.safetensors +2026-09-08 11:25:08,562 [Rank 15] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00002-of-00002.safetensors +2026-09-08 11:25:26,458 [Rank 15] 811,965,856 state entries loaded successfully +2026-09-08 11:25:29,450 [Rank 15] Data loading took 1,057.23 ms +2026-09-08 12:27:57,595 [Rank 15] Data loading took 45,500.37 ms +2026-09-08 12:28:37,863 [Rank 15] Data loading took 13,361.33 ms diff --git a/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_16.txt b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_16.txt new file mode 100644 index 0000000000000000000000000000000000000000..dbd284efb047930b80c9f1d3b4c515915a8e4020 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_16.txt @@ -0,0 +1,7 @@ +2026-09-08 11:16:32,877 [Rank 16] Loading index from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model.safetensors.index.json +2026-09-08 11:16:32,883 [Rank 16] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00001-of-00002.safetensors +2026-09-08 11:25:08,568 [Rank 16] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00002-of-00002.safetensors +2026-09-08 11:25:26,458 [Rank 16] 811,965,856 state entries loaded successfully +2026-09-08 11:25:29,461 [Rank 16] Data loading took 1,069.14 ms +2026-09-08 12:27:57,595 [Rank 16] Data loading took 45,500.77 ms +2026-09-08 12:28:37,863 [Rank 16] Data loading took 13,361.22 ms diff --git a/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_17.txt b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_17.txt new file mode 100644 index 0000000000000000000000000000000000000000..23271d8680805e9742132959754037e7eecc1d38 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_17.txt @@ -0,0 +1,7 @@ +2026-09-08 11:16:32,877 [Rank 17] Loading index from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model.safetensors.index.json +2026-09-08 11:16:32,884 [Rank 17] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00001-of-00002.safetensors +2026-09-08 11:25:08,568 [Rank 17] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00002-of-00002.safetensors +2026-09-08 11:25:26,458 [Rank 17] 811,965,856 state entries loaded successfully +2026-09-08 11:25:29,464 [Rank 17] Data loading took 1,057.75 ms +2026-09-08 12:27:57,595 [Rank 17] Data loading took 45,500.54 ms +2026-09-08 12:28:37,863 [Rank 17] Data loading took 13,361.15 ms diff --git a/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_18.txt b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_18.txt new file mode 100644 index 0000000000000000000000000000000000000000..27bd09489d8d9ac8a474c846e31a8c724a21a9bd --- /dev/null +++ b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_18.txt @@ -0,0 +1,7 @@ +2026-09-08 11:16:32,871 [Rank 18] Loading index from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model.safetensors.index.json +2026-09-08 11:16:32,877 [Rank 18] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00001-of-00002.safetensors +2026-09-08 11:25:08,568 [Rank 18] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00002-of-00002.safetensors +2026-09-08 11:25:26,458 [Rank 18] 811,965,856 state entries loaded successfully +2026-09-08 11:25:29,442 [Rank 18] Data loading took 1,052.85 ms +2026-09-08 12:27:57,595 [Rank 18] Data loading took 45,500.49 ms +2026-09-08 12:28:37,863 [Rank 18] Data loading took 13,361.13 ms diff --git a/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_19.txt b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_19.txt new file mode 100644 index 0000000000000000000000000000000000000000..09e918da56b3011bbfa6d29cfbb04062c3d16c1b --- /dev/null +++ b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_19.txt @@ -0,0 +1,7 @@ +2026-09-08 11:16:32,868 [Rank 19] Loading index from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model.safetensors.index.json +2026-09-08 11:16:32,875 [Rank 19] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00001-of-00002.safetensors +2026-09-08 11:25:08,568 [Rank 19] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00002-of-00002.safetensors +2026-09-08 11:25:26,458 [Rank 19] 811,965,856 state entries loaded successfully +2026-09-08 11:25:29,460 [Rank 19] Data loading took 1,055.78 ms +2026-09-08 12:27:57,595 [Rank 19] Data loading took 45,500.14 ms +2026-09-08 12:28:37,863 [Rank 19] Data loading took 13,361.02 ms diff --git a/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_20.txt b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_20.txt new file mode 100644 index 0000000000000000000000000000000000000000..17e344714b776b271af348a7e3deede2b4338c4f --- /dev/null +++ b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_20.txt @@ -0,0 +1,7 @@ +2026-09-08 11:16:32,855 [Rank 20] Loading index from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model.safetensors.index.json +2026-09-08 11:16:32,863 [Rank 20] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00001-of-00002.safetensors +2026-09-08 11:25:08,568 [Rank 20] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00002-of-00002.safetensors +2026-09-08 11:25:26,458 [Rank 20] 811,965,856 state entries loaded successfully +2026-09-08 11:25:29,454 [Rank 20] Data loading took 1,051.26 ms +2026-09-08 12:27:57,595 [Rank 20] Data loading took 45,500.57 ms +2026-09-08 12:28:37,863 [Rank 20] Data loading took 13,361.29 ms diff --git a/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_21.txt b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_21.txt new file mode 100644 index 0000000000000000000000000000000000000000..a584b24f14c32c05e6c04efd9076fbb44079985e --- /dev/null +++ b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_21.txt @@ -0,0 +1,7 @@ +2026-09-08 11:16:32,861 [Rank 21] Loading index from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model.safetensors.index.json +2026-09-08 11:16:32,868 [Rank 21] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00001-of-00002.safetensors +2026-09-08 11:25:08,568 [Rank 21] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00002-of-00002.safetensors +2026-09-08 11:25:26,458 [Rank 21] 811,965,856 state entries loaded successfully +2026-09-08 11:25:29,443 [Rank 21] Data loading took 1,043.72 ms +2026-09-08 12:27:57,595 [Rank 21] Data loading took 45,500.67 ms +2026-09-08 12:28:37,863 [Rank 21] Data loading took 13,361.36 ms diff --git a/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_22.txt b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_22.txt new file mode 100644 index 0000000000000000000000000000000000000000..07ef4f8804920a1cc637ac938779baa5ac7ff9de --- /dev/null +++ b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_22.txt @@ -0,0 +1,7 @@ +2026-09-08 11:16:32,863 [Rank 22] Loading index from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model.safetensors.index.json +2026-09-08 11:16:32,870 [Rank 22] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00001-of-00002.safetensors +2026-09-08 11:25:08,568 [Rank 22] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00002-of-00002.safetensors +2026-09-08 11:25:26,458 [Rank 22] 811,965,856 state entries loaded successfully +2026-09-08 11:25:29,431 [Rank 22] Data loading took 1,036.63 ms +2026-09-08 12:27:57,595 [Rank 22] Data loading took 45,500.16 ms +2026-09-08 12:28:37,863 [Rank 22] Data loading took 13,361.26 ms diff --git a/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_23.txt b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_23.txt new file mode 100644 index 0000000000000000000000000000000000000000..81c0539f6ad80714b5c1497522e009ad04efd040 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_23.txt @@ -0,0 +1,7 @@ +2026-09-08 11:16:32,878 [Rank 23] Loading index from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model.safetensors.index.json +2026-09-08 11:16:32,884 [Rank 23] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00001-of-00002.safetensors +2026-09-08 11:25:08,568 [Rank 23] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00002-of-00002.safetensors +2026-09-08 11:25:26,458 [Rank 23] 811,965,856 state entries loaded successfully +2026-09-08 11:25:29,442 [Rank 23] Data loading took 1,040.56 ms +2026-09-08 12:27:57,595 [Rank 23] Data loading took 45,500.08 ms +2026-09-08 12:28:37,863 [Rank 23] Data loading took 13,361.72 ms diff --git a/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_24.txt b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_24.txt new file mode 100644 index 0000000000000000000000000000000000000000..34565d9853f19ea07e7e156ea9b0ac9939d5f5f8 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_24.txt @@ -0,0 +1,7 @@ +2026-09-08 11:16:32,870 [Rank 24] Loading index from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model.safetensors.index.json +2026-09-08 11:16:32,876 [Rank 24] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00001-of-00002.safetensors +2026-09-08 11:25:08,564 [Rank 24] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00002-of-00002.safetensors +2026-09-08 11:25:26,458 [Rank 24] 811,965,856 state entries loaded successfully +2026-09-08 11:25:29,467 [Rank 24] Data loading took 1,083.28 ms +2026-09-08 12:27:57,596 [Rank 24] Data loading took 45,500.88 ms +2026-09-08 12:28:37,863 [Rank 24] Data loading took 13,361.21 ms diff --git a/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_25.txt b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_25.txt new file mode 100644 index 0000000000000000000000000000000000000000..62963040582fb8681263172d1cb133cc4a26d7a7 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_25.txt @@ -0,0 +1,7 @@ +2026-09-08 11:16:32,868 [Rank 25] Loading index from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model.safetensors.index.json +2026-09-08 11:16:32,874 [Rank 25] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00001-of-00002.safetensors +2026-09-08 11:25:08,563 [Rank 25] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00002-of-00002.safetensors +2026-09-08 11:25:26,458 [Rank 25] 811,965,856 state entries loaded successfully +2026-09-08 11:25:29,467 [Rank 25] Data loading took 1,073.43 ms +2026-09-08 12:27:57,595 [Rank 25] Data loading took 45,500.50 ms +2026-09-08 12:28:37,863 [Rank 25] Data loading took 13,360.95 ms diff --git a/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_26.txt b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_26.txt new file mode 100644 index 0000000000000000000000000000000000000000..03d950f9681b5d542de00915b6a9a049d85425fe --- /dev/null +++ b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_26.txt @@ -0,0 +1,7 @@ +2026-09-08 11:16:32,858 [Rank 26] Loading index from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model.safetensors.index.json +2026-09-08 11:16:32,865 [Rank 26] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00001-of-00002.safetensors +2026-09-08 11:25:08,563 [Rank 26] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00002-of-00002.safetensors +2026-09-08 11:25:26,458 [Rank 26] 811,965,856 state entries loaded successfully +2026-09-08 11:25:29,455 [Rank 26] Data loading took 1,069.86 ms +2026-09-08 12:27:57,595 [Rank 26] Data loading took 45,500.44 ms +2026-09-08 12:28:37,863 [Rank 26] Data loading took 13,361.03 ms diff --git a/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_27.txt b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_27.txt new file mode 100644 index 0000000000000000000000000000000000000000..f88546975d03f499f631e2ba72ecaca7f2f18295 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_27.txt @@ -0,0 +1,7 @@ +2026-09-08 11:16:32,860 [Rank 27] Loading index from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model.safetensors.index.json +2026-09-08 11:16:32,867 [Rank 27] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00001-of-00002.safetensors +2026-09-08 11:25:08,563 [Rank 27] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00002-of-00002.safetensors +2026-09-08 11:25:26,458 [Rank 27] 811,965,856 state entries loaded successfully +2026-09-08 11:25:29,474 [Rank 27] Data loading took 1,074.57 ms +2026-09-08 12:27:57,595 [Rank 27] Data loading took 45,500.19 ms +2026-09-08 12:28:37,863 [Rank 27] Data loading took 13,361.60 ms diff --git a/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_28.txt b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_28.txt new file mode 100644 index 0000000000000000000000000000000000000000..fed96c6dc5171200fd4c1fdcdabbf90e64af9918 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_28.txt @@ -0,0 +1,7 @@ +2026-09-08 11:16:32,870 [Rank 28] Loading index from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model.safetensors.index.json +2026-09-08 11:16:32,877 [Rank 28] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00001-of-00002.safetensors +2026-09-08 11:25:08,564 [Rank 28] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00002-of-00002.safetensors +2026-09-08 11:25:26,458 [Rank 28] 811,965,856 state entries loaded successfully +2026-09-08 11:25:29,470 [Rank 28] Data loading took 1,085.48 ms +2026-09-08 12:27:57,595 [Rank 28] Data loading took 45,500.57 ms +2026-09-08 12:28:37,863 [Rank 28] Data loading took 13,361.49 ms diff --git a/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_29.txt b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_29.txt new file mode 100644 index 0000000000000000000000000000000000000000..8469c02bda782434d4edbab0e8f16a12b3c7ce40 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_29.txt @@ -0,0 +1,7 @@ +2026-09-08 11:16:32,870 [Rank 29] Loading index from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model.safetensors.index.json +2026-09-08 11:16:32,876 [Rank 29] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00001-of-00002.safetensors +2026-09-08 11:25:08,564 [Rank 29] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00002-of-00002.safetensors +2026-09-08 11:25:26,458 [Rank 29] 811,965,856 state entries loaded successfully +2026-09-08 11:25:29,459 [Rank 29] Data loading took 1,061.06 ms +2026-09-08 12:27:57,595 [Rank 29] Data loading took 45,500.64 ms +2026-09-08 12:28:37,863 [Rank 29] Data loading took 13,361.31 ms diff --git a/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_30.txt b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_30.txt new file mode 100644 index 0000000000000000000000000000000000000000..60d668f639e75ecfd4b905c5963648b412cb0bb0 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_30.txt @@ -0,0 +1,7 @@ +2026-09-08 11:16:32,857 [Rank 30] Loading index from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model.safetensors.index.json +2026-09-08 11:16:32,865 [Rank 30] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00001-of-00002.safetensors +2026-09-08 11:25:08,564 [Rank 30] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00002-of-00002.safetensors +2026-09-08 11:25:26,458 [Rank 30] 811,965,856 state entries loaded successfully +2026-09-08 11:25:29,443 [Rank 30] Data loading took 1,058.77 ms +2026-09-08 12:27:57,595 [Rank 30] Data loading took 45,500.26 ms +2026-09-08 12:28:37,863 [Rank 30] Data loading took 13,361.39 ms diff --git a/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_31.txt b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_31.txt new file mode 100644 index 0000000000000000000000000000000000000000..c31794ef5b3c5469909d483abb73c9ae09eb1dba --- /dev/null +++ b/26_09_08_sft_email_v4_v3/runs/0/logs/logs_rank_31.txt @@ -0,0 +1,7 @@ +2026-09-08 11:16:32,860 [Rank 31] Loading index from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model.safetensors.index.json +2026-09-08 11:16:32,866 [Rank 31] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00001-of-00002.safetensors +2026-09-08 11:25:08,564 [Rank 31] Loading from /mnt/queue1/shruthan/models/gemma-4-26B-A4B-it-causal-lm/model-00002-of-00002.safetensors +2026-09-08 11:25:26,458 [Rank 31] 811,965,856 state entries loaded successfully +2026-09-08 11:25:29,454 [Rank 31] Data loading took 1,069.99 ms +2026-09-08 12:27:57,595 [Rank 31] Data loading took 45,500.33 ms +2026-09-08 12:28:37,863 [Rank 31] Data loading took 13,361.23 ms diff --git a/26_09_08_sft_email_v4_v3/wandb_config.yaml b/26_09_08_sft_email_v4_v3/wandb_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..83f0a7dd21cbe034303b1878a0a88ef4c3d5ea77 --- /dev/null +++ b/26_09_08_sft_email_v4_v3/wandb_config.yaml @@ -0,0 +1,7 @@ +entity: corellm +group: default +id: adsk4wbby19ihe1l +name: 26_09_08_sft_email_v4_v3 +project: gemma-sft +resume: allow +save_code: false