essentialweb_raw/2000/model_config.json

{"bos_token_id": 128000, "eos_token_id": 128001, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 6144, "is_qwen2_config": true, "max_position_embeddings": 4096, "num_attention_heads": 16, "num_hidden_layers": 28, "num_key_value_heads": 8, "pad_token_id": null, "pretraining_tp": 1, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 10000, "rope_interleaved": false, "tie_word_embeddings": true, "use_cache": true, "vocab_size": 128256, "_attn_implementation": "flash_attention_2", "flex_attention_mask": null, "attention_bias": false, "sliding_window_size": null, "z_loss_enabled": false, "z_loss_coefficient": 1e-05, "no_rope_layer": null, "_fused_rotary_emb": true, "_fused_rms_norm": true, "_use_qkv_packed": true, "_use_doc_masking": true, "moe_config": null}