| _target_: nemo.collections.llm.gpt.model.hyena.HyenaModel |
| config: |
| _cpu_offloading_context: null |
| _target_: nemo.collections.llm.gpt.model.hyena.Hyena7bConfig |
| account_for_embedding_in_pipeline_split: false |
| account_for_loss_in_pipeline_split: false |
| activation_func: |
| _call_: false |
| _target_: torch._C._nn.gelu |
| activation_func_fp8_input_store: false |
| add_attn_proj_bias: true |
| add_bias_linear: false |
| add_bias_output: false |
| add_qkv_bias: false |
| apply_query_key_layer_scaling: false |
| apply_residual_connection_post_layernorm: false |
| apply_rope_fusion: true |
| async_tensor_model_parallel_allreduce: false |
| attention_backend: |
| _call_: true |
| _target_: megatron.core.transformer.enums.AttnBackend |
| attention_dropout: 0.0 |
| attention_softmax_in_fp32: true |
| autocast_dtype: null |
| barrier_with_L1_time: true |
| batch_p2p_comm: true |
| batch_p2p_sync: true |
| bf16: true |
| bias_activation_fusion: true |
| bias_dropout_add_fusion: true |
| bias_dropout_fusion: false |
| calculate_per_token_loss: false |
| clone_scatter_output_in_embedding: true |
| config_logger_dir: '' |
| context_parallel_size: 1 |
| cp_comm_type: null |
| cpu_offloading: false |
| cpu_offloading_activations: true |
| cpu_offloading_num_layers: 0 |
| cpu_offloading_weights: true |
| cross_entropy_fusion_impl: native |
| cross_entropy_loss_fusion: false |
| cuda_graph_retain_backward_graph: false |
| cuda_graph_scope: full |
| cuda_graph_use_single_mempool: false |
| cuda_graph_warmup_steps: 3 |
| data_step_fn: |
| _call_: false |
| _target_: nemo.collections.llm.gpt.model.base.gpt_data_step |
| deallocate_pipeline_outputs: false |
| defer_embedding_wgrad_compute: false |
| delay_wgrad_compute: false |
| deterministic_mode: false |
| disable_bf16_reduced_precision_matmul: false |
| disable_parameter_transpose_cache: false |
| distribute_saved_activations: null |
| embedding_init_method: null |
| embedding_init_method_std: null |
| enable_autocast: false |
| enable_cuda_graph: false |
| expert_model_parallel_size: 1 |
| expert_tensor_parallel_size: null |
| external_cuda_graph: false |
| ffn_hidden_size: 11008 |
| finalize_model_grads_func: null |
| first_last_layers_bf16: false |
| flash_decode: false |
| forward_step_fn: |
| _call_: false |
| _target_: nemo.collections.llm.gpt.model.hyena.hyena_forward_step |
| fp16: false |
| fp16_lm_cross_entropy: false |
| fp32_residual_connection: true |
| fp8: null |
| fp8_amax_compute_algo: most_recent |
| fp8_amax_history_len: 1 |
| fp8_dot_product_attention: false |
| fp8_interval: 1 |
| fp8_margin: 0 |
| fp8_multi_head_attention: false |
| fp8_param: false |
| fp8_recipe: delayed |
| fp8_wgrad: true |
| gated_linear_unit: true |
| get_attention_mask_from_fusion: false |
| grad_scale_func: null |
| grad_sync_func: null |
| gradient_accumulation_fusion: false |
| hetereogenous_dist_checkpoint: false |
| heterogeneous_block_specs: false |
| hidden_dropout: 0.0 |
| hidden_size: 4096 |
| hierarchical_context_parallel_sizes: null |
| hybrid_attention_ratio: 0.0 |
| hybrid_mlp_ratio: 0.0 |
| hybrid_override_pattern: SDH*SDHSDH*SDHSDH*SDHSDH*SDHSDH* |
| hyena_filter_no_wd: true |
| hyena_init_method: small_init |
| hyena_output_layer_init_method: wang_init |
| inference_rng_tracker: false |
| init_method: null |
| init_method_std: 0.02 |
| init_model_with_meta_device: false |
| is_hybrid_model: false |
| kv_channels: null |
| layernorm_epsilon: 1.0e-06 |
| layernorm_zero_centered_gamma: false |
| make_vocab_size_divisible_by: 8 |
| mamba_head_dim: 64 |
| mamba_num_groups: 8 |
| mamba_num_heads: null |
| mamba_state_dim: 128 |
| mapping_type: base |
| masked_softmax_fusion: false |
| memory_efficient_layer_norm: false |
| microbatch_group_size_per_vp_stage: 1 |
| mlp_chunks_for_prefill: 1 |
| moe_apply_probs_on_input: false |
| moe_aux_loss_coeff: 0 |
| moe_deepep_num_sms: 20 |
| moe_enable_deepep: false |
| moe_expert_capacity_factor: null |
| moe_extended_tp: false |
| moe_ffn_hidden_size: null |
| moe_grouped_gemm: false |
| moe_input_jitter_eps: null |
| moe_layer_freq: 1 |
| moe_layer_recompute: false |
| moe_pad_expert_input_to_capacity: false |
| moe_per_layer_logging: false |
| moe_permute_fusion: false |
| moe_router_bias_update_rate: 0.001 |
| moe_router_dtype: null |
| moe_router_enable_expert_bias: false |
| moe_router_force_load_balancing: false |
| moe_router_group_topk: null |
| moe_router_load_balancing_type: aux_loss |
| moe_router_num_groups: null |
| moe_router_padding_for_fp8: false |
| moe_router_pre_softmax: false |
| moe_router_score_function: softmax |
| moe_router_topk: 2 |
| moe_router_topk_limited_devices: null |
| moe_router_topk_scaling_factor: null |
| moe_shared_expert_intermediate_size: null |
| moe_shared_expert_overlap: false |
| moe_token_dispatcher_type: allgather |
| moe_token_drop_policy: probs |
| moe_token_dropping: false |
| moe_use_legacy_grouped_gemm: false |
| moe_z_loss_coeff: null |
| mrope_section: null |
| mtp_loss_scaling_factor: null |
| mtp_num_layers: null |
| multi_latent_attention: false |
| no_rope_freq: null |
| no_sync_func: null |
| normalization: RMSNorm |
| num_attention_heads: 32 |
| num_groups_hyena: 4096 |
| num_groups_hyena_medium: 256 |
| num_groups_hyena_short: 256 |
| num_layers: 32 |
| num_layers_at_end_in_bf16: 1 |
| num_layers_at_start_in_bf16: 1 |
| num_layers_in_first_pipeline_stage: null |
| num_layers_in_last_pipeline_stage: null |
| num_microbatches_with_partial_activation_checkpoints: null |
| num_moe_experts: null |
| num_query_groups: null |
| output_layer_init_method: null |
| overlap_p2p_comm: false |
| overlap_p2p_comm_warmup_flush: false |
| parallel_output: true |
| param_sync_func: null |
| params_dtype: |
| _call_: false |
| _target_: torch.bfloat16 |
| perform_initialization: true |
| persist_layer_norm: false |
| pipeline_dtype: null |
| pipeline_model_parallel_comm_backend: null |
| pipeline_model_parallel_layout: null |
| pipeline_model_parallel_size: 1 |
| pipeline_model_parallel_split_rank: null |
| position_embedding_type: rope |
| post_process: true |
| pre_process: true |
| qk_layernorm: false |
| quant_recipe: null |
| recompute_granularity: full |
| recompute_method: uniform |
| recompute_modules: null |
| recompute_num_layers: 4 |
| remove_activation_post_first_layer: true |
| rotary_base: 10000 |
| rotary_interleaved: false |
| rotary_percent: 1.0 |
| seq_len_interpolation_factor: null |
| seq_length: 8192 |
| sequence_parallel: false |
| softmax_scale: null |
| symmetric_ar_type: null |
| tensor_model_parallel_size: 1 |
| test_mode: false |
| timers: null |
| to_upper: normalized_weighted |
| tokenizer_library: byte-level |
| tokenizer_model_path: null |
| tp_comm_atomic_ag: false |
| tp_comm_atomic_rs: false |
| tp_comm_bootstrap_backend: nccl |
| tp_comm_bulk_dgrad: true |
| tp_comm_bulk_wgrad: true |
| tp_comm_overlap: false |
| tp_comm_overlap_ag: true |
| tp_comm_overlap_disable_fc1: false |
| tp_comm_overlap_disable_qkv: false |
| tp_comm_overlap_rs: true |
| tp_comm_overlap_rs_dgrad: false |
| tp_comm_split_ag: true |
| tp_comm_split_rs: true |
| tp_only_amax_red: false |
| use_b2b_causal_conv1d: false |
| use_cpu_initialization: false |
| use_kitchen: false |
| use_mamba_mem_eff_path: true |
| use_ring_exchange_p2p: false |
| use_short_conv_bias: false |
| use_te: true |
| use_te_rng_tracker: false |
| variable_seq_lengths: false |
| virtual_pipeline_model_parallel_size: null |
| vortex_style_fp8: false |
| wgrad_deferral_limit: 0 |
| window_size: null |
| model_context_managers: [] |
| model_transform: null |
| optim: |
| _target_: nemo.lightning.pytorch.optim.megatron.MegatronOptimizerModule |
| config: |
| _target_: megatron.core.optimizer.optimizer_config.OptimizerConfig |
| adam_beta1: 0.9 |
| adam_beta2: 0.999 |
| adam_eps: 1.0e-08 |
| barrier_with_L1_time: false |
| bf16: false |
| clip_grad: 1.0 |
| config_logger_dir: '' |
| decoupled_lr: null |
| decoupled_min_lr: null |
| exp_avg_dtype: |
| _call_: false |
| _target_: torch.float32 |
| exp_avg_sq_dtype: |
| _call_: false |
| _target_: torch.float32 |
| fp16: false |
| fp8_recipe: null |
| hysteresis: 2 |
| initial_loss_scale: 4294967296 |
| log_num_zeros_in_grad: false |
| loss_scale: null |
| loss_scale_window: 1000 |
| lr: 0.0001 |
| main_grads_dtype: |
| _call_: false |
| _target_: torch.float32 |
| main_params_dtype: |
| _call_: false |
| _target_: torch.float32 |
| min_loss_scale: 1.0 |
| min_lr: null |
| optimizer: adam |
| optimizer_cpu_offload: false |
| optimizer_offload_fraction: 0.0 |
| overlap_cpu_optimizer_d2h_h2d: false |
| overlap_param_gather_with_optimizer_step: false |
| params_dtype: |
| _call_: false |
| _target_: torch.float32 |
| pin_cpu_grads: true |
| pin_cpu_params: true |
| reuse_grad_buf_for_mxfp8_param_ag: false |
| sgd_momentum: 0.9 |
| store_param_remainders: true |
| timers: null |
| use_distributed_optimizer: true |
| use_precision_aware_optimizer: false |
| use_torch_optimizer_for_cpu_offload: false |
| weight_decay: 0.01 |
| lr_mult: 1.0 |
| lr_scheduler: null |
| no_weight_decay_cond: null |
| scale_lr_cond: null |
| tokenizer: |
| _bos_id: null |
| _eos_id: 0 |
| _pad_id: 1 |
| _target_: nemo.collections.common.tokenizers.bytelevel_tokenizers.ByteLevelTokenizer |
| special_tokens: {} |
| vocab_size: 512 |
|
|
