_target_: nemo.collections.llm.gpt.model.hyena.HyenaModel
config:
  _cpu_offloading_context: null
  _target_: nemo.collections.llm.gpt.model.hyena.Hyena7bConfig
  account_for_embedding_in_pipeline_split: false
  account_for_loss_in_pipeline_split: false
  activation_func:
    _call_: false
    _target_: torch._C._nn.gelu
  activation_func_fp8_input_store: false
  add_attn_proj_bias: true
  add_bias_linear: false
  add_bias_output: false
  add_qkv_bias: false
  apply_query_key_layer_scaling: false
  apply_residual_connection_post_layernorm: false
  apply_rope_fusion: true
  async_tensor_model_parallel_allreduce: false
  attention_backend:
    _call_: true
    _target_: megatron.core.transformer.enums.AttnBackend
  attention_dropout: 0.0
  attention_softmax_in_fp32: true
  autocast_dtype: null
  barrier_with_L1_time: true
  batch_p2p_comm: true
  batch_p2p_sync: true
  bf16: true
  bias_activation_fusion: true
  bias_dropout_add_fusion: true
  bias_dropout_fusion: false
  calculate_per_token_loss: false
  clone_scatter_output_in_embedding: true
  config_logger_dir: ''
  context_parallel_size: 1
  cp_comm_type: null
  cpu_offloading: false
  cpu_offloading_activations: true
  cpu_offloading_num_layers: 0
  cpu_offloading_weights: true
  cross_entropy_fusion_impl: native
  cross_entropy_loss_fusion: false
  cuda_graph_retain_backward_graph: false
  cuda_graph_scope: full
  cuda_graph_use_single_mempool: false
  cuda_graph_warmup_steps: 3
  data_step_fn:
    _call_: false
    _target_: nemo.collections.llm.gpt.model.base.gpt_data_step
  deallocate_pipeline_outputs: false
  defer_embedding_wgrad_compute: false
  delay_wgrad_compute: false
  deterministic_mode: false
  disable_bf16_reduced_precision_matmul: false
  disable_parameter_transpose_cache: false
  distribute_saved_activations: null
  embedding_init_method: null
  embedding_init_method_std: null
  enable_autocast: false
  enable_cuda_graph: false
  expert_model_parallel_size: 1
  expert_tensor_parallel_size: null
  external_cuda_graph: false
  ffn_hidden_size: 11008
  finalize_model_grads_func: null
  first_last_layers_bf16: false
  flash_decode: false
  forward_step_fn:
    _call_: false
    _target_: nemo.collections.llm.gpt.model.hyena.hyena_forward_step
  fp16: false
  fp16_lm_cross_entropy: false
  fp32_residual_connection: true
  fp8: null
  fp8_amax_compute_algo: most_recent
  fp8_amax_history_len: 1
  fp8_dot_product_attention: false
  fp8_interval: 1
  fp8_margin: 0
  fp8_multi_head_attention: false
  fp8_param: false
  fp8_recipe: delayed
  fp8_wgrad: true
  gated_linear_unit: true
  get_attention_mask_from_fusion: false
  grad_scale_func: null
  grad_sync_func: null
  gradient_accumulation_fusion: false
  hetereogenous_dist_checkpoint: false
  heterogeneous_block_specs: false
  hidden_dropout: 0.0
  hidden_size: 4096
  hierarchical_context_parallel_sizes: null
  hybrid_attention_ratio: 0.0
  hybrid_mlp_ratio: 0.0
  hybrid_override_pattern: SDH*SDHSDH*SDHSDH*SDHSDH*SDHSDH*
  hyena_filter_no_wd: true
  hyena_init_method: small_init
  hyena_output_layer_init_method: wang_init
  inference_rng_tracker: false
  init_method: null
  init_method_std: 0.02
  init_model_with_meta_device: false
  is_hybrid_model: false
  kv_channels: null
  layernorm_epsilon: 1.0e-06
  layernorm_zero_centered_gamma: false
  make_vocab_size_divisible_by: 8
  mamba_head_dim: 64
  mamba_num_groups: 8
  mamba_num_heads: null
  mamba_state_dim: 128
  mapping_type: base
  masked_softmax_fusion: false
  memory_efficient_layer_norm: false
  microbatch_group_size_per_vp_stage: 1
  mlp_chunks_for_prefill: 1
  moe_apply_probs_on_input: false
  moe_aux_loss_coeff: 0
  moe_deepep_num_sms: 20
  moe_enable_deepep: false
  moe_expert_capacity_factor: null
  moe_extended_tp: false
  moe_ffn_hidden_size: null
  moe_grouped_gemm: false
  moe_input_jitter_eps: null
  moe_layer_freq: 1
  moe_layer_recompute: false
  moe_pad_expert_input_to_capacity: false
  moe_per_layer_logging: false
  moe_permute_fusion: false
  moe_router_bias_update_rate: 0.001
  moe_router_dtype: null
  moe_router_enable_expert_bias: false
  moe_router_force_load_balancing: false
  moe_router_group_topk: null
  moe_router_load_balancing_type: aux_loss
  moe_router_num_groups: null
  moe_router_padding_for_fp8: false
  moe_router_pre_softmax: false
  moe_router_score_function: softmax
  moe_router_topk: 2
  moe_router_topk_limited_devices: null
  moe_router_topk_scaling_factor: null
  moe_shared_expert_intermediate_size: null
  moe_shared_expert_overlap: false
  moe_token_dispatcher_type: allgather
  moe_token_drop_policy: probs
  moe_token_dropping: false
  moe_use_legacy_grouped_gemm: false
  moe_z_loss_coeff: null
  mrope_section: null
  mtp_loss_scaling_factor: null
  mtp_num_layers: null
  multi_latent_attention: false
  no_rope_freq: null
  no_sync_func: null
  normalization: RMSNorm
  num_attention_heads: 32
  num_groups_hyena: 4096
  num_groups_hyena_medium: 256
  num_groups_hyena_short: 256
  num_layers: 32
  num_layers_at_end_in_bf16: 1
  num_layers_at_start_in_bf16: 1
  num_layers_in_first_pipeline_stage: null
  num_layers_in_last_pipeline_stage: null
  num_microbatches_with_partial_activation_checkpoints: null
  num_moe_experts: null
  num_query_groups: null
  output_layer_init_method: null
  overlap_p2p_comm: false
  overlap_p2p_comm_warmup_flush: false
  parallel_output: true
  param_sync_func: null
  params_dtype:
    _call_: false
    _target_: torch.bfloat16
  perform_initialization: true
  persist_layer_norm: false
  pipeline_dtype: null
  pipeline_model_parallel_comm_backend: null
  pipeline_model_parallel_layout: null
  pipeline_model_parallel_size: 1
  pipeline_model_parallel_split_rank: null
  position_embedding_type: rope
  post_process: true
  pre_process: true
  qk_layernorm: false
  quant_recipe: null
  recompute_granularity: full
  recompute_method: uniform
  recompute_modules: null
  recompute_num_layers: 4
  remove_activation_post_first_layer: true
  rotary_base: 10000
  rotary_interleaved: false
  rotary_percent: 1.0
  seq_len_interpolation_factor: null
  seq_length: 8192
  sequence_parallel: false
  softmax_scale: null
  symmetric_ar_type: null
  tensor_model_parallel_size: 1
  test_mode: false
  timers: null
  to_upper: normalized_weighted
  tokenizer_library: byte-level
  tokenizer_model_path: null
  tp_comm_atomic_ag: false
  tp_comm_atomic_rs: false
  tp_comm_bootstrap_backend: nccl
  tp_comm_bulk_dgrad: true
  tp_comm_bulk_wgrad: true
  tp_comm_overlap: false
  tp_comm_overlap_ag: true
  tp_comm_overlap_disable_fc1: false
  tp_comm_overlap_disable_qkv: false
  tp_comm_overlap_rs: true
  tp_comm_overlap_rs_dgrad: false
  tp_comm_split_ag: true
  tp_comm_split_rs: true
  tp_only_amax_red: false
  use_b2b_causal_conv1d: false
  use_cpu_initialization: false
  use_kitchen: false
  use_mamba_mem_eff_path: true
  use_ring_exchange_p2p: false
  use_short_conv_bias: false
  use_te: true
  use_te_rng_tracker: false
  variable_seq_lengths: false
  virtual_pipeline_model_parallel_size: null
  vortex_style_fp8: false
  wgrad_deferral_limit: 0
  window_size: null
model_context_managers: []
model_transform: null
optim:
  _target_: nemo.lightning.pytorch.optim.megatron.MegatronOptimizerModule
  config:
    _target_: megatron.core.optimizer.optimizer_config.OptimizerConfig
    adam_beta1: 0.9
    adam_beta2: 0.999
    adam_eps: 1.0e-08
    barrier_with_L1_time: false
    bf16: false
    clip_grad: 1.0
    config_logger_dir: ''
    decoupled_lr: null
    decoupled_min_lr: null
    exp_avg_dtype:
      _call_: false
      _target_: torch.float32
    exp_avg_sq_dtype:
      _call_: false
      _target_: torch.float32
    fp16: false
    fp8_recipe: null
    hysteresis: 2
    initial_loss_scale: 4294967296
    log_num_zeros_in_grad: false
    loss_scale: null
    loss_scale_window: 1000
    lr: 0.0001
    main_grads_dtype:
      _call_: false
      _target_: torch.float32
    main_params_dtype:
      _call_: false
      _target_: torch.float32
    min_loss_scale: 1.0
    min_lr: null
    optimizer: adam
    optimizer_cpu_offload: false
    optimizer_offload_fraction: 0.0
    overlap_cpu_optimizer_d2h_h2d: false
    overlap_param_gather_with_optimizer_step: false
    params_dtype:
      _call_: false
      _target_: torch.float32
    pin_cpu_grads: true
    pin_cpu_params: true
    reuse_grad_buf_for_mxfp8_param_ag: false
    sgd_momentum: 0.9
    store_param_remainders: true
    timers: null
    use_distributed_optimizer: true
    use_precision_aware_optimizer: false
    use_torch_optimizer_for_cpu_offload: false
    weight_decay: 0.01
  lr_mult: 1.0
  lr_scheduler: null
  no_weight_decay_cond: null
  scale_lr_cond: null
tokenizer:
  _bos_id: null
  _eos_id: 0
  _pad_id: 1
  _target_: nemo.collections.common.tokenizers.bytelevel_tokenizers.ByteLevelTokenizer
  special_tokens: {}
  vocab_size: 512