File size: 8,867 Bytes
cfbc06c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 | _target_: nemo.collections.llm.gpt.model.hyena.HyenaModel
config:
_cpu_offloading_context: null
_target_: nemo.collections.llm.gpt.model.hyena.Hyena7bConfig
account_for_embedding_in_pipeline_split: false
account_for_loss_in_pipeline_split: false
activation_func:
_call_: false
_target_: torch._C._nn.gelu
activation_func_fp8_input_store: false
add_attn_proj_bias: true
add_bias_linear: false
add_bias_output: false
add_qkv_bias: false
apply_query_key_layer_scaling: false
apply_residual_connection_post_layernorm: false
apply_rope_fusion: true
async_tensor_model_parallel_allreduce: false
attention_backend:
_call_: true
_target_: megatron.core.transformer.enums.AttnBackend
attention_dropout: 0.0
attention_softmax_in_fp32: true
autocast_dtype: null
barrier_with_L1_time: true
batch_p2p_comm: true
batch_p2p_sync: true
bf16: true
bias_activation_fusion: true
bias_dropout_add_fusion: true
bias_dropout_fusion: false
calculate_per_token_loss: false
clone_scatter_output_in_embedding: true
config_logger_dir: ''
context_parallel_size: 1
cp_comm_type: null
cpu_offloading: false
cpu_offloading_activations: true
cpu_offloading_num_layers: 0
cpu_offloading_weights: true
cross_entropy_fusion_impl: native
cross_entropy_loss_fusion: false
cuda_graph_retain_backward_graph: false
cuda_graph_scope: full
cuda_graph_use_single_mempool: false
cuda_graph_warmup_steps: 3
data_step_fn:
_call_: false
_target_: nemo.collections.llm.gpt.model.base.gpt_data_step
deallocate_pipeline_outputs: false
defer_embedding_wgrad_compute: false
delay_wgrad_compute: false
deterministic_mode: false
disable_bf16_reduced_precision_matmul: false
disable_parameter_transpose_cache: false
distribute_saved_activations: null
embedding_init_method: null
embedding_init_method_std: null
enable_autocast: false
enable_cuda_graph: false
expert_model_parallel_size: 1
expert_tensor_parallel_size: null
external_cuda_graph: false
ffn_hidden_size: 11008
finalize_model_grads_func: null
first_last_layers_bf16: false
flash_decode: false
forward_step_fn:
_call_: false
_target_: nemo.collections.llm.gpt.model.hyena.hyena_forward_step
fp16: false
fp16_lm_cross_entropy: false
fp32_residual_connection: true
fp8: null
fp8_amax_compute_algo: most_recent
fp8_amax_history_len: 1
fp8_dot_product_attention: false
fp8_interval: 1
fp8_margin: 0
fp8_multi_head_attention: false
fp8_param: false
fp8_recipe: delayed
fp8_wgrad: true
gated_linear_unit: true
get_attention_mask_from_fusion: false
grad_scale_func: null
grad_sync_func: null
gradient_accumulation_fusion: false
hetereogenous_dist_checkpoint: false
heterogeneous_block_specs: false
hidden_dropout: 0.0
hidden_size: 4096
hierarchical_context_parallel_sizes: null
hybrid_attention_ratio: 0.0
hybrid_mlp_ratio: 0.0
hybrid_override_pattern: SDH*SDHSDH*SDHSDH*SDHSDH*SDHSDH*
hyena_filter_no_wd: true
hyena_init_method: small_init
hyena_output_layer_init_method: wang_init
inference_rng_tracker: false
init_method: null
init_method_std: 0.02
init_model_with_meta_device: false
is_hybrid_model: false
kv_channels: null
layernorm_epsilon: 1.0e-06
layernorm_zero_centered_gamma: false
make_vocab_size_divisible_by: 8
mamba_head_dim: 64
mamba_num_groups: 8
mamba_num_heads: null
mamba_state_dim: 128
mapping_type: base
masked_softmax_fusion: false
memory_efficient_layer_norm: false
microbatch_group_size_per_vp_stage: 1
mlp_chunks_for_prefill: 1
moe_apply_probs_on_input: false
moe_aux_loss_coeff: 0
moe_deepep_num_sms: 20
moe_enable_deepep: false
moe_expert_capacity_factor: null
moe_extended_tp: false
moe_ffn_hidden_size: null
moe_grouped_gemm: false
moe_input_jitter_eps: null
moe_layer_freq: 1
moe_layer_recompute: false
moe_pad_expert_input_to_capacity: false
moe_per_layer_logging: false
moe_permute_fusion: false
moe_router_bias_update_rate: 0.001
moe_router_dtype: null
moe_router_enable_expert_bias: false
moe_router_force_load_balancing: false
moe_router_group_topk: null
moe_router_load_balancing_type: aux_loss
moe_router_num_groups: null
moe_router_padding_for_fp8: false
moe_router_pre_softmax: false
moe_router_score_function: softmax
moe_router_topk: 2
moe_router_topk_limited_devices: null
moe_router_topk_scaling_factor: null
moe_shared_expert_intermediate_size: null
moe_shared_expert_overlap: false
moe_token_dispatcher_type: allgather
moe_token_drop_policy: probs
moe_token_dropping: false
moe_use_legacy_grouped_gemm: false
moe_z_loss_coeff: null
mrope_section: null
mtp_loss_scaling_factor: null
mtp_num_layers: null
multi_latent_attention: false
no_rope_freq: null
no_sync_func: null
normalization: RMSNorm
num_attention_heads: 32
num_groups_hyena: 4096
num_groups_hyena_medium: 256
num_groups_hyena_short: 256
num_layers: 32
num_layers_at_end_in_bf16: 1
num_layers_at_start_in_bf16: 1
num_layers_in_first_pipeline_stage: null
num_layers_in_last_pipeline_stage: null
num_microbatches_with_partial_activation_checkpoints: null
num_moe_experts: null
num_query_groups: null
output_layer_init_method: null
overlap_p2p_comm: false
overlap_p2p_comm_warmup_flush: false
parallel_output: true
param_sync_func: null
params_dtype:
_call_: false
_target_: torch.bfloat16
perform_initialization: true
persist_layer_norm: false
pipeline_dtype: null
pipeline_model_parallel_comm_backend: null
pipeline_model_parallel_layout: null
pipeline_model_parallel_size: 1
pipeline_model_parallel_split_rank: null
position_embedding_type: rope
post_process: true
pre_process: true
qk_layernorm: false
quant_recipe: null
recompute_granularity: full
recompute_method: uniform
recompute_modules: null
recompute_num_layers: 4
remove_activation_post_first_layer: true
rotary_base: 10000
rotary_interleaved: false
rotary_percent: 1.0
seq_len_interpolation_factor: null
seq_length: 8192
sequence_parallel: false
softmax_scale: null
symmetric_ar_type: null
tensor_model_parallel_size: 1
test_mode: false
timers: null
to_upper: normalized_weighted
tokenizer_library: byte-level
tokenizer_model_path: null
tp_comm_atomic_ag: false
tp_comm_atomic_rs: false
tp_comm_bootstrap_backend: nccl
tp_comm_bulk_dgrad: true
tp_comm_bulk_wgrad: true
tp_comm_overlap: false
tp_comm_overlap_ag: true
tp_comm_overlap_disable_fc1: false
tp_comm_overlap_disable_qkv: false
tp_comm_overlap_rs: true
tp_comm_overlap_rs_dgrad: false
tp_comm_split_ag: true
tp_comm_split_rs: true
tp_only_amax_red: false
use_b2b_causal_conv1d: false
use_cpu_initialization: false
use_kitchen: false
use_mamba_mem_eff_path: true
use_ring_exchange_p2p: false
use_short_conv_bias: false
use_te: true
use_te_rng_tracker: false
variable_seq_lengths: false
virtual_pipeline_model_parallel_size: null
vortex_style_fp8: false
wgrad_deferral_limit: 0
window_size: null
model_context_managers: []
model_transform: null
optim:
_target_: nemo.lightning.pytorch.optim.megatron.MegatronOptimizerModule
config:
_target_: megatron.core.optimizer.optimizer_config.OptimizerConfig
adam_beta1: 0.9
adam_beta2: 0.999
adam_eps: 1.0e-08
barrier_with_L1_time: false
bf16: false
clip_grad: 1.0
config_logger_dir: ''
decoupled_lr: null
decoupled_min_lr: null
exp_avg_dtype:
_call_: false
_target_: torch.float32
exp_avg_sq_dtype:
_call_: false
_target_: torch.float32
fp16: false
fp8_recipe: null
hysteresis: 2
initial_loss_scale: 4294967296
log_num_zeros_in_grad: false
loss_scale: null
loss_scale_window: 1000
lr: 0.0001
main_grads_dtype:
_call_: false
_target_: torch.float32
main_params_dtype:
_call_: false
_target_: torch.float32
min_loss_scale: 1.0
min_lr: null
optimizer: adam
optimizer_cpu_offload: false
optimizer_offload_fraction: 0.0
overlap_cpu_optimizer_d2h_h2d: false
overlap_param_gather_with_optimizer_step: false
params_dtype:
_call_: false
_target_: torch.float32
pin_cpu_grads: true
pin_cpu_params: true
reuse_grad_buf_for_mxfp8_param_ag: false
sgd_momentum: 0.9
store_param_remainders: true
timers: null
use_distributed_optimizer: true
use_precision_aware_optimizer: false
use_torch_optimizer_for_cpu_offload: false
weight_decay: 0.01
lr_mult: 1.0
lr_scheduler: null
no_weight_decay_cond: null
scale_lr_cond: null
tokenizer:
_bos_id: null
_eos_id: 0
_pad_id: 1
_target_: nemo.collections.common.tokenizers.bytelevel_tokenizers.ByteLevelTokenizer
special_tokens: {}
vocab_size: 512
|